db: store cumulative rct output distribution in the db for speed

This gets rid of the temporary precalc cache.

Also make the RPC able to send data back in binary or JSON,
since there can be a lot of data

This bumps the LMDB database format to v3, with migration.

9 files changed, 315 insertions, 177 deletions

diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp
index 88ac34255..8544cc3f0 100644
--- a/src/blockchain_db/blockchain_db.cpp
+++ b/src/blockchain_db/blockchain_db.cpp
@@ -218,13 +218,22 @@ uint64_t BlockchainDB::add_block( const block& blk
   // call out to add the transactions
 
   time1 = epee::misc_utils::get_tick_count();
+
+  uint64_t num_rct_outs = 0;
   add_transaction(blk_hash, blk.miner_tx);
+  if (blk.miner_tx.version == 2)
+    num_rct_outs += blk.miner_tx.vout.size();
   int tx_i = 0;
   crypto::hash tx_hash = crypto::null_hash;
   for (const transaction& tx : txs)
   {
     tx_hash = blk.tx_hashes[tx_i];
     add_transaction(blk_hash, tx, &tx_hash);
+    for (const auto &vout: tx.vout)
+    {
+      if (vout.amount == 0)
+        ++num_rct_outs;
+    }
     ++tx_i;
   }
   TIME_MEASURE_FINISH(time1);
@@ -232,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk
 
   // call out to subclass implementation to add the block & metadata
   time1 = epee::misc_utils::get_tick_count();
-  add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash);
+  add_block(blk, block_size, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash);
   TIME_MEASURE_FINISH(time1);
   time_add_block1 += time1;
 
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
index 564016fc9..a52782b3c 100644
--- a/src/blockchain_db/blockchain_db.h
+++ b/src/blockchain_db/blockchain_db.h
@@ -367,6 +367,7 @@ private:
                 , const size_t& block_size
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
+                , uint64_t num_rct_outs
                 , const crypto::hash& blk_hash
                 ) = 0;
 
@@ -892,6 +893,20 @@ public:
   virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0;
 
   /**
+   * @brief fetch a block's cumulative number of rct outputs
+   *
+   * The subclass should return the numer of rct outputs in the blockchain
+   * up to the block with the given height (inclusive).
+   *
+   * If the block does not exist, the subclass should throw BLOCK_DNE
+   *
+   * @param height the height requested
+   *
+   * @return the cumulative number of rct outputs
+   */
+  virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const = 0;
+
+  /**
    * @brief fetch the top block's timestamp
    *
    * The subclass should return the timestamp of the most recent block.
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index 367bfa49e..3aaade3b4 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -53,7 +53,7 @@ using epee::string_tools::pod_to_hex;
 using namespace crypto;
 
 // Increase when the DB structure changes
-#define VERSION 2
+#define VERSION 3
 
 namespace
 {
@@ -250,6 +250,16 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi
 namespace cryptonote
 {
 
+typedef struct mdb_block_info_old
+{
+  uint64_t bi_height;
+  uint64_t bi_timestamp;
+  uint64_t bi_coins;
+  uint64_t bi_size; // a size_t really but we need 32-bit compat
+  difficulty_type bi_diff;
+  crypto::hash bi_hash;
+} mdb_block_info_old;
+
 typedef struct mdb_block_info
 {
   uint64_t bi_height;
@@ -258,6 +268,7 @@ typedef struct mdb_block_info
   uint64_t bi_size; // a size_t really but we need 32-bit compat
   difficulty_type bi_diff;
   crypto::hash bi_hash;
+  uint64_t bi_cum_rct;
 } mdb_block_info;
 
 typedef struct blk_height {
@@ -667,7 +678,7 @@ estim:
 }
 
 void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
-    const crypto::hash& blk_hash)
+    uint64_t num_rct_outs, const crypto::hash& blk_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -715,6 +726,16 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   bi.bi_size = block_size;
   bi.bi_diff = cumulative_difficulty;
   bi.bi_hash = blk_hash;
+  bi.bi_cum_rct = num_rct_outs;
+  if (blk.major_version >= 4)
+  {
+    uint64_t last_height = m_height-1;
+    MDB_val_set(h, last_height);
+    if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH)))
+        throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str()));
+    const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data;
+    bi.bi_cum_rct += bi_prev->bi_cum_rct;
+  }
 
   MDB_val_set(val, bi);
   result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP);
@@ -759,8 +780,6 @@ void BlockchainLMDB::remove_block()
   if ((result = mdb_cursor_del(m_cur_block_heights, 0)))
       throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str()));
 
-  if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET)))
-      throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str()));
   if ((result = mdb_cursor_del(m_cur_blocks, 0)))
       throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str()));
 
@@ -1143,6 +1162,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB()
   m_cum_size = 0;
   m_cum_count = 0;
 
+  // reset may also need changing when initialize things here
+
   m_hardfork = nullptr;
 }
 
@@ -1908,6 +1929,43 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
   return ret;
 }
 
+std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  std::vector<uint64_t> res;
+  int result;
+
+  if (heights.empty())
+    return {};
+  res.reserve(heights.size());
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(block_info);
+
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_blocks, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+  for (size_t i = 0; i < heights.size(); ++i)
+    if (heights[i] >= db_stats.ms_entries)
+      throw0(BLOCK_DNE(std::string("Attempt to get rct distribution from height " + std::to_string(heights[i]) + " failed -- block size not in db").c_str()));
+
+  MDB_val v;
+
+  for (uint64_t height: heights)
+  {
+    MDB_val_set(v, height);
+    result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str()));
+    const mdb_block_info *bi = (const mdb_block_info *)v.mv_data;
+    res.push_back(bi->bi_cum_rct);
+  }
+
+  TXN_POSTFIX_RDONLY();
+  return res;
+}
+
 uint64_t BlockchainLMDB::get_top_block_timestamp() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -3328,6 +3386,7 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig
   MDB_val_set(k, amount);
   MDB_val v;
   MDB_cursor_op op = MDB_SET;
+  base = 0;
   while (1)
   {
     int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
@@ -3346,6 +3405,9 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig
       break;
   }
 
+  for (size_t n = 1; n < distribution.size(); ++n)
+    distribution[n] += distribution[n - 1];
+
   TXN_POSTFIX_RDONLY();
 
   return true;
@@ -3440,7 +3502,7 @@ void BlockchainLMDB::fixup()
     if (result) \
       throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \
     ptr = (char *)k.mv_data; \
-    ptr[sizeof(name)-2] = 's'
+    ptr[sizeof(name)-2]++
 
 #define LOGIF(y)    if (ELPP->vRegistry()->allowed(y, "global"))
 
@@ -3580,7 +3642,7 @@ void BlockchainLMDB::migrate_0_1()
       break;
     }
     MDB_dbi diffs, hashes, sizes, timestamps;
-    mdb_block_info bi;
+    mdb_block_info_old bi;
     MDB_val_set(nv, bi);
 
     lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs");
@@ -4120,6 +4182,141 @@ void BlockchainLMDB::migrate_1_2()
   txn.commit();
 }
 
+void BlockchainLMDB::migrate_2_3()
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  uint64_t i;
+  int result;
+  mdb_txn_safe txn(false);
+  MDB_val k, v;
+  char *ptr;
+
+  MGINFO_YELLOW("Migrating blockchain from DB version 2 to 3 - this may take a while:");
+
+  do {
+    LOG_PRINT_L1("migrating block info:");
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+
+    MDB_stat db_stats;
+    if ((result = mdb_stat(txn, m_blocks, &db_stats)))
+      throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+    const uint64_t blockchain_height = db_stats.ms_entries;
+
+    MDEBUG("enumerating rct outputs...");
+    std::vector<uint64_t> distribution(blockchain_height, 0);
+    bool r = for_all_outputs(0, [&](uint64_t height) {
+      if (height >= blockchain_height)
+      {
+        MERROR("Output found claiming height >= blockchain height");
+        return false;
+      }
+      distribution[height]++;
+      return true;
+    });
+    if (!r)
+      throw0(DB_ERROR("Failed to build rct output distribution"));
+    for (size_t i = 1; i < distribution.size(); ++i)
+      distribution[i] += distribution[i - 1];
+
+    /* the block_info table name is the same but the old version and new version
+     * have incompatible data. Create a new table. We want the name to be similar
+     * to the old name so that it will occupy the same location in the DB.
+     */
+    MDB_dbi o_block_info = m_block_info;
+    lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+    MDB_cursor *c_old, *c_cur;
+    i = 0;
+    while(1) {
+      if (!(i % 1000)) {
+        if (i) {
+          LOGIF(el::Level::Info) {
+            std::cout << i << " / " << blockchain_height << "  \r" << std::flush;
+          }
+          txn.commit();
+          result = mdb_txn_begin(m_env, NULL, 0, txn);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+        }
+        result = mdb_cursor_open(txn, m_block_info, &c_cur);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str()));
+        result = mdb_cursor_open(txn, o_block_info, &c_old);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
+        if (!i) {
+          MDB_stat db_stat;
+          result = mdb_stat(txn, m_block_info, &db_stats);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str()));
+          i = db_stats.ms_entries;
+        }
+      }
+      result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
+      if (result == MDB_NOTFOUND) {
+        txn.commit();
+        break;
+      }
+      else if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str()));
+      const mdb_block_info_old *bi_old = (const mdb_block_info_old*)v.mv_data;
+      mdb_block_info bi;
+      bi.bi_height = bi_old->bi_height;
+      bi.bi_timestamp = bi_old->bi_timestamp;
+      bi.bi_coins = bi_old->bi_coins;
+      bi.bi_size = bi_old->bi_size;
+      bi.bi_diff = bi_old->bi_diff;
+      bi.bi_hash = bi_old->bi_hash;
+      if (bi_old->bi_height >= distribution.size())
+        throw0(DB_ERROR("Bad height in block_info record"));
+      bi.bi_cum_rct = distribution[bi_old->bi_height];
+      MDB_val_set(nv, bi);
+      result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str()));
+      /* we delete the old records immediately, so the overall DB and mapsize should not grow.
+       * This is a little slower than just letting mdb_drop() delete it all at the end, but
+       * it saves a significant amount of disk space.
+       */
+      result = mdb_cursor_del(c_old, 0);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str()));
+      i++;
+    }
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+    /* Delete the old table */
+    result = mdb_drop(txn, o_block_info, 1);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str()));
+
+    RENAME_DB("block_infn");
+
+    lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+    txn.commit();
+  } while(0);
+
+  uint32_t version = 3;
+  v.mv_data = (void *)&version;
+  v.mv_size = sizeof(version);
+  MDB_val_copy<const char *> vk("version");
+  result = mdb_txn_begin(m_env, NULL, 0, txn);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+  result = mdb_put(txn, m_properties, &vk, &v, 0);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
+  txn.commit();
+}
+
 void BlockchainLMDB::migrate(const uint32_t oldversion)
 {
   switch(oldversion) {
@@ -4127,6 +4324,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion)
     migrate_0_1(); /* FALLTHRU */
   case 1:
     migrate_1_2(); /* FALLTHRU */
+  case 2:
+    migrate_2_3(); /* FALLTHRU */
   default:
     ;
   }
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index cc1b06ca0..b7f6262ae 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -197,6 +197,8 @@ public:
 
   virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const;
 
+  virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const;
+
   virtual uint64_t get_block_timestamp(const uint64_t& height) const;
 
   virtual uint64_t get_top_block_timestamp() const;
@@ -317,6 +319,7 @@ private:
                 , const size_t& block_size
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
+                , uint64_t num_rct_outs
                 , const crypto::hash& block_hash
                 );
 
@@ -387,6 +390,9 @@ private:
   // migrate from DB version 1 to 2
   void migrate_1_2();
 
+  // migrate from DB version 2 to 3
+  void migrate_2_3();
+
   void cleanup_batch();
 
 private:
diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp
index 73ce98366..a7cd0947a 100644
--- a/src/cryptonote_core/blockchain.cpp
+++ b/src/cryptonote_core/blockchain.cpp
@@ -1985,14 +1985,14 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint
 //------------------------------------------------------------------
 bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const
 {
-  // rct outputs don't exist before v3
+  // rct outputs don't exist before v4
   if (amount == 0)
   {
     switch (m_nettype)
     {
-      case STAGENET: start_height = stagenet_hard_forks[2].height; break;
-      case TESTNET: start_height = testnet_hard_forks[2].height; break;
-      case MAINNET: start_height = mainnet_hard_forks[2].height; break;
+      case STAGENET: start_height = stagenet_hard_forks[3].height; break;
+      case TESTNET: start_height = testnet_hard_forks[3].height; break;
+      case MAINNET: start_height = mainnet_hard_forks[3].height; break;
       default: return false;
     }
   }
@@ -2000,11 +2000,40 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height,
     start_height = 0;
   base = 0;
 
+  if (to_height > 0 && to_height < from_height)
+    return false;
+
   const uint64_t real_start_height = start_height;
   if (from_height > start_height)
     start_height = from_height;
 
-  return m_db->get_output_distribution(amount, start_height, to_height, distribution, base);
+  distribution.clear();
+  uint64_t db_height = m_db->height();
+  if (db_height == 0)
+    return false;
+  if (to_height == 0)
+    to_height = db_height - 1;
+  if (start_height >= db_height || to_height >= db_height)
+    return false;
+  if (amount == 0)
+  {
+    std::vector<uint64_t> heights;
+    heights.reserve(to_height + 1 - start_height);
+    uint64_t real_start_height = start_height > 0 ? start_height-1 : start_height;
+    for (uint64_t h = real_start_height; h <= to_height; ++h)
+      heights.push_back(h);
+    distribution = m_db->get_block_cumulative_rct_outputs(heights);
+    if (start_height > 0)
+    {
+      base = distribution[0];
+      distribution.erase(distribution.begin());
+    }
+    return true;
+  }
+  else
+  {
+    return m_db->get_output_distribution(amount, start_height, to_height, distribution, base);
+  }
 }
 //------------------------------------------------------------------
 // This function takes a list of block hashes from another node
diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp
index b55b1994b..857370dbb 100644
--- a/src/rpc/core_rpc_server.cpp
+++ b/src/rpc/core_rpc_server.cpp
@@ -47,7 +47,6 @@ using namespace epee;
 #include "rpc/rpc_args.h"
 #include "core_rpc_server_error_codes.h"
 #include "p2p/net_node.h"
-#include "get_output_distribution_cache.h"
 #include "version.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -2071,6 +2070,10 @@ namespace cryptonote
   bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp)
   {
     PERF_TIMER(on_get_output_distribution);
+    bool r;
+    if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r))
+      return r;
+
     try
     {
       for (uint64_t amount: req.amounts)
@@ -2087,38 +2090,17 @@ namespace cryptonote
 
         if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req.to_height)
         {
-          res.distributions.push_back({amount, d.cached_start_height, d.cached_distribution, d.cached_base});
-          if (req.cumulative)
+          res.distributions.push_back({amount, d.cached_start_height, req.binary, d.cached_distribution, d.cached_base});
+          if (!req.cumulative)
           {
             auto &distribution = res.distributions.back().distribution;
-            distribution[0] += d.cached_base;
-            for (size_t n = 1; n < distribution.size(); ++n)
-              distribution[n] += distribution[n-1];
+            for (size_t n = distribution.size() - 1; n > 0; --n)
+              distribution[n] -= distribution[n-1];
+            distribution[0] -= d.cached_base;
           }
           continue;
         }
 
-        // this is a slow operation, so we have precomputed caches of common cases
-        bool found = false;
-        for (const auto &slot: get_output_distribution_cache)
-        {
-          if (slot.amount == amount && slot.from_height == req.from_height && slot.to_height == req.to_height)
-          {
-            res.distributions.push_back({amount, slot.start_height, slot.distribution, slot.base});
-            found = true;
-            if (req.cumulative)
-            {
-              auto &distribution = res.distributions.back().distribution;
-              distribution[0] += slot.base;
-              for (size_t n = 1; n < distribution.size(); ++n)
-                distribution[n] += distribution[n-1];
-            }
-            break;
-          }
-        }
-        if (found)
-          continue;
-
         std::vector<uint64_t> distribution;
         uint64_t start_height, base;
         if (!m_core.get_output_distribution(amount, req.from_height, req.to_height, start_height, distribution, base))
@@ -2144,14 +2126,14 @@ namespace cryptonote
           d.cached = true;
         }
 
-        if (req.cumulative)
+        if (!req.cumulative)
         {
-          distribution[0] += base;
-          for (size_t n = 1; n < distribution.size(); ++n)
-            distribution[n] += distribution[n-1];
+          for (size_t n = distribution.size() - 1; n > 0; --n)
+            distribution[n] -= distribution[n-1];
+          distribution[0] -= base;
         }
 
-        res.distributions.push_back({amount, start_height, std::move(distribution), base});
+        res.distributions.push_back({amount, start_height, req.binary, std::move(distribution), base});
       }
     }
     catch (const std::exception &e)
diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h
index 49b730149..05690390d 100644
--- a/src/rpc/core_rpc_server_commands_defs.h
+++ b/src/rpc/core_rpc_server_commands_defs.h
@@ -49,7 +49,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 1
-#define CORE_RPC_VERSION_MINOR 20
+#define CORE_RPC_VERSION_MINOR 21
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -2231,12 +2231,14 @@ namespace cryptonote
       uint64_t from_height;
       uint64_t to_height;
       bool cumulative;
+      bool binary;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(amounts)
         KV_SERIALIZE_OPT(from_height, (uint64_t)0)
         KV_SERIALIZE_OPT(to_height, (uint64_t)0)
         KV_SERIALIZE_OPT(cumulative, false)
+        KV_SERIALIZE_OPT(binary, true)
       END_KV_SERIALIZE_MAP()
     };
 
@@ -2244,13 +2246,18 @@ namespace cryptonote
     {
       uint64_t amount;
       uint64_t start_height;
+      bool binary;
       std::vector<uint64_t> distribution;
       uint64_t base;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(amount)
         KV_SERIALIZE(start_height)
-        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution)
+        KV_SERIALIZE(binary)
+        if (this_ref.binary)
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution)
+        else
+          KV_SERIALIZE(distribution)
         KV_SERIALIZE(base)
       END_KV_SERIALIZE_MAP()
     };
@@ -2259,10 +2266,12 @@ namespace cryptonote
     {
       std::string status;
       std::vector<distribution> distributions;
+      bool untrusted;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(status)
         KV_SERIALIZE(distributions)
+        KV_SERIALIZE(untrusted)
       END_KV_SERIALIZE_MAP()
     };
   };
diff --git a/src/rpc/get_output_distribution_cache.h b/src/rpc/get_output_distribution_cache.h
deleted file mode 100644
index 6495e7d4c..000000000
--- a/src/rpc/get_output_distribution_cache.h
+++ /dev/null
@@ -1,113 +0,0 @@
-static const struct
-{
-  uint64_t amount;
-  uint64_t from_height;
-  uint64_t to_height;
-  uint64_t start_height;
-  uint64_t base;
-  std::vector<uint64_t> distribution;
-}
-get_output_distribution_cache[] =
-{
-  {
-    0,
-    1544704,
-    1546001,
-    1544704,
-    5143500,
-    {
-      5, 38, 37, 33, 39, 7, 1, 1, 5, 9, 7, 5, 17, 5, 3, 9, 3, 17, 5, 17, 1, 1, 15, 13, 3, 10, 5, 3, 34, 1, 45, 7, 
-      5, 17, 5, 22, 3, 1, 17, 16, 5, 1, 3, 43, 5, 13, 3, 23, 9, 7, 9, 13, 1, 11, 1, 17, 1, 3, 16, 11, 5, 11, 7, 7, 
-      33, 11, 7, 1, 5, 1, 21, 19, 1, 17, 1, 49, 17, 3, 3, 9, 35, 46, 46, 39, 26, 33, 21, 3, 23, 3, 9, 37, 1, 33, 11, 32, 
-      1, 13, 16, 12, 3, 21, 1, 18, 3, 19, 1, 25, 5, 3, 18, 7, 17, 5, 9, 15, 7, 7, 11, 9, 9, 17, 5, 16, 1, 3, 13, 3, 
-      5, 5, 5, 13, 5, 9, 5, 13, 3, 17, 15, 36, 13, 3, 20, 12, 6, 23, 17, 10, 22, 23, 1, 7, 21, 6, 23, 1, 3, 19, 13, 1, 
-      3, 43, 35, 13, 1, 31, 7, 3, 17, 1, 15, 5, 11, 15, 24, 1, 18, 13, 5, 15, 1, 29, 3, 3, 13, 3, 15, 7, 17, 3, 1, 1, 
-      17, 1, 1, 45, 39, 27, 45, 46, 34, 7, 3, 3, 9, 3, 3, 11, 7, 5, 9, 25, 19, 3, 33, 1, 5, 17, 1, 45, 4, 1, 45, 11, 
-      44, 32, 3, 1, 3, 7, 17, 15, 5, 45, 35, 41, 1, 35, 3, 3, 19, 1, 9, 17, 29, 29, 3, 1, 13, 1, 3, 47, 21, 13, 7, 1, 
-      7, 5, 1, 11, 1, 40, 9, 7, 3, 3, 13, 25, 1, 47, 5, 7, 3, 7, 31, 40, 34, 6, 3, 15, 3, 31, 5, 13, 27, 9, 12, 21, 
-      3, 1, 19, 1, 19, 5, 47, 49, 47, 42, 50, 34, 29, 23, 1, 5, 9, 16, 11, 7, 1, 19, 7, 5, 1, 15, 1, 1, 9, 13, 9, 5, 
-      27, 3, 3, 29, 1, 33, 3, 9, 5, 35, 5, 1, 17, 7, 3, 39, 3, 28, 19, 1, 1, 9, 1, 3, 27, 1, 37, 3, 1, 1, 16, 3, 
-      25, 11, 5, 3, 33, 45, 17, 11, 7, 22, 9, 1, 5, 5, 5, 15, 1, 15, 9, 7, 11, 13, 37, 49, 46, 38, 11, 1, 25, 1, 13, 18, 
-      3, 7, 39, 3, 37, 19, 35, 3, 1, 3, 19, 1, 3, 15, 21, 3, 27, 1, 45, 48, 1, 13, 29, 9, 1, 1, 46, 43, 5, 15, 3, 7, 
-      29, 26, 5, 5, 21, 37, 17, 21, 3, 13, 1, 5, 1, 17, 5, 31, 13, 1, 11, 3, 46, 9, 3, 7, 1, 1, 41, 1, 21, 1, 5, 12, 
-      7, 13, 9, 25, 1, 47, 47, 48, 48, 48, 48, 48, 47, 48, 45, 45, 33, 52, 50, 46, 45, 47, 35, 41, 38, 35, 42, 38, 34, 41, 39, 35, 
-      51, 51, 45, 43, 49, 52, 53, 45, 42, 46, 37, 53, 49, 41, 46, 49, 46, 47, 48, 37, 41, 33, 43, 38, 15, 3, 3, 27, 11, 5, 23, 13, 
-      1, 1, 37, 3, 15, 3, 30, 13, 3, 45, 12, 3, 5, 11, 1, 1, 21, 9, 11, 19, 1, 1, 1, 25, 5, 21, 3, 1, 32, 44, 3, 33, 
-      11, 7, 5, 23, 1, 37, 47, 48, 48, 48, 48, 48, 48, 46, 47, 47, 50, 45, 49, 50, 46, 47, 49, 45, 51, 49, 50, 49, 49, 46, 47, 48, 
-      46, 48, 46, 50, 46, 43, 46, 46, 48, 47, 46, 47, 45, 49, 46, 43, 50, 45, 45, 49, 45, 48, 45, 49, 48, 45, 45, 51, 45, 51, 45, 46, 
-      52, 45, 45, 51, 51, 52, 44, 45, 52, 50, 50, 46, 47, 51, 51, 46, 47, 47, 47, 50, 47, 51, 48, 49, 51, 50, 48, 48, 48, 50, 49, 49, 
-      52, 52, 49, 50, 49, 49, 49, 51, 52, 49, 52, 50, 49, 47, 29, 15, 39, 17, 31, 5, 40, 5, 18, 23, 25, 7, 35, 26, 5, 31, 49, 22, 
-      3, 17, 7, 49, 7, 49, 47, 12, 44, 46, 36, 15, 3, 1, 47, 13, 35, 40, 5, 21, 19, 39, 21, 33, 31, 29, 1, 1, 37, 1, 15, 47, 
-      7, 7, 47, 41, 13, 3, 47, 31, 9, 33, 13, 43, 29, 5, 1, 9, 33, 7, 27, 15, 15, 25, 5, 43, 22, 31, 7, 1, 47, 1, 15, 27, 
-      3, 27, 45, 15, 1, 36, 17, 1, 23, 39, 38, 45, 7, 7, 19, 7, 11, 47, 33, 16, 3, 45, 45, 45, 9, 27, 3, 3, 21, 3, 7, 21, 
-      7, 3, 43, 1, 17, 1, 45, 37, 46, 5, 5, 13, 46, 40, 48, 48, 45, 34, 1, 46, 19, 25, 9, 7, 47, 23, 37, 31, 3, 25, 13, 46, 
-      31, 25, 5, 46, 35, 52, 11, 23, 27, 4, 15, 11, 11, 11, 9, 34, 7, 9, 15, 34, 9, 27, 37, 28, 25, 45, 13, 30, 5, 25, 15, 7, 
-      3, 19, 27, 1, 7, 11, 1, 32, 3, 45, 11, 9, 21, 25, 9, 13, 13, 1, 7, 1, 33, 11, 5, 3, 3, 27, 27, 5, 3, 37, 17, 17, 
-      3, 7, 5, 13, 1, 3, 44, 45, 26, 25, 1, 13, 3, 13, 3, 11, 1, 11, 7, 45, 3, 3, 1, 43, 1, 19, 3, 1, 15, 5, 39, 7, 
-      7, 1, 9, 1, 11, 19, 3, 35, 29, 7, 15, 11, 40, 7, 44, 38, 34, 7, 9, 7, 1, 27, 1, 9, 5, 45, 1, 21, 3, 1, 5, 9, 
-      3, 21, 23, 33, 3, 1, 7, 3, 3, 7, 41, 9, 7, 1, 5, 31, 9, 7, 1, 1, 11, 41, 51, 20, 9, 47, 39, 17, 9, 35, 1, 41, 
-      1, 19, 1, 19, 15, 1, 13, 5, 23, 15, 9, 15, 17, 1, 15, 27, 33, 31, 29, 7, 13, 1, 5, 45, 5, 1, 1, 11, 1, 13, 3, 7, 
-      9, 1, 13, 39, 3, 33, 5, 3, 7, 7, 5, 29, 11, 1, 7, 1, 15, 3, 13, 3, 15, 3, 3, 1, 5, 1, 9, 1, 44, 49, 24, 25, 
-      1, 1, 34, 22, 7, 5, 5, 5, 10, 9, 13, 3, 9, 1, 9, 19, 7, 43, 48, 7, 11, 7, 3, 3, 7, 21, 1, 1, 3, 3, 11, 31, 
-      1, 1, 13, 22, 23, 7, 27, 9, 3, 3, 21, 1, 35, 21, 9, 11, 13, 39, 1, 3, 7, 23, 3, 28, 3, 45, 47, 38, 32, 37, 34, 1, 
-      23, 3, 3, 1, 19, 19, 1, 5, 13, 1, 5, 11, 38, 3, 1, 36, 13, 1, 1, 23, 5, 17, 11, 1, 13, 1, 3, 7, 11, 3, 33, 7, 
-      19, 5, 5, 1, 1, 3, 5, 41, 1, 3, 25, 1, 7, 7, 9, 3, 11, 3, 13, 5, 7, 1, 3, 9, 1, 1, 43, 47, 47, 47, 17, 7, 
-      17, 3, 19, 1, 9, 9, 33, 22, 1, 25, 1, 3, 3, 32, 5, 1, 23, 9, 5, 1, 31, 5, 9, 1, 3, 7, 19, 1, 12, 11, 5, 1, 
-      1, 9, 25, 15, 15, 13, 5, 3, 15, 1, 17, 1, 1, 5, 5, 41, 11, 15, 7, 3, 21, 21, 35, 22, 46, 35, 3, 27, 5, 3, 45, 22, 
-      27, 1, 19, 9, 1, 25, 1, 29, 3, 5, 25, 17, 27, 5, 19, 5, 25, 7, 19, 1, 9, 21, 3, 7, 29, 27, 17, 3, 3, 15, 7, 19, 
-      5, 25, 1, 23, 45, 4, 31, 1, 37, 14, 29, 3, 29, 1, 23, 29, 19, 11, 1, 13, 13, 9, 1, 25, 1, 33, 1, 37, 37, 23, 7, 21, 
-      7, 3, 13, 7, 3, 7, 21, 11, 9, 1, 31, 3, 1, 7, 39, 46, 3, 30, 
-    },
-  },
-  {
-    0,
-    1562704,
-    1564001,
-    1562704,
-    5521986,
-    {
-      35, 45, 23, 3, 44, 47, 44, 3, 17, 35, 7, 11, 7, 29, 43, 27, 11, 7, 5, 31, 13, 9, 45, 45, 7, 42, 17, 15, 19, 11, 45, 19, 
-      45, 46, 45, 46, 32, 34, 43, 34, 46, 47, 45, 30, 17, 45, 46, 36, 35, 38, 19, 9, 23, 17, 3, 19, 31, 41, 35, 24, 15, 45, 15, 5, 
-      11, 5, 19, 11, 11, 7, 15, 19, 45, 34, 7, 7, 29, 1, 9, 36, 7, 44, 45, 33, 25, 8, 17, 7, 44, 43, 48, 45, 42, 46, 40, 44, 
-      1, 43, 45, 46, 46, 35, 19, 19, 23, 5, 13, 19, 7, 16, 9, 3, 25, 34, 3, 27, 9, 39, 3, 43, 21, 1, 45, 45, 39, 25, 23, 13, 
-      39, 39, 3, 45, 43, 46, 44, 40, 39, 33, 45, 47, 38, 45, 45, 39, 47, 47, 45, 46, 35, 45, 43, 47, 45, 40, 34, 42, 42, 48, 49, 47, 
-      47, 48, 47, 45, 43, 48, 37, 48, 41, 45, 48, 34, 42, 44, 9, 19, 27, 1, 47, 47, 43, 25, 29, 5, 5, 21, 39, 35, 43, 37, 13, 45, 
-      25, 31, 26, 47, 45, 23, 23, 39, 32, 25, 44, 47, 35, 47, 15, 17, 7, 9, 5, 35, 31, 3, 45, 47, 46, 13, 17, 48, 45, 9, 13, 45, 
-      45, 31, 1, 53, 44, 46, 39, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 47, 47, 47, 47, 47, 47, 47, 53, 49, 45, 45, 50, 
-      50, 27, 43, 46, 47, 46, 45, 48, 36, 42, 42, 46, 45, 47, 41, 45, 43, 47, 38, 48, 47, 33, 11, 45, 46, 34, 42, 32, 3, 45, 37, 45, 
-      15, 3, 45, 29, 31, 9, 5, 3, 27, 5, 21, 25, 7, 15, 46, 34, 5, 3, 17, 3, 9, 13, 7, 11, 3, 1, 34, 13, 7, 45, 33, 26, 
-      9, 5, 9, 41, 43, 34, 3, 35, 3, 17, 37, 11, 17, 3, 15, 27, 15, 45, 46, 13, 26, 16, 11, 7, 5, 45, 38, 45, 45, 22, 44, 44, 
-      43, 6, 11, 35, 15, 44, 17, 27, 13, 3, 40, 5, 9, 7, 35, 19, 5, 5, 1, 28, 33, 15, 45, 5, 29, 3, 31, 12, 5, 32, 24, 3, 
-      23, 13, 47, 45, 42, 46, 39, 21, 21, 1, 44, 44, 47, 41, 5, 1, 11, 36, 20, 5, 45, 39, 45, 45, 44, 45, 32, 22, 40, 11, 38, 1, 
-      45, 46, 37, 9, 23, 9, 15, 44, 7, 16, 38, 46, 11, 14, 24, 19, 19, 7, 26, 25, 45, 37, 17, 1, 35, 1, 3, 28, 3, 11, 22, 13, 
-      3, 1, 7, 38, 5, 3, 1, 26, 1, 3, 43, 44, 46, 45, 21, 11, 1, 44, 27, 1, 11, 26, 10, 44, 45, 45, 45, 47, 47, 45, 48, 45, 
-      38, 9, 5, 44, 46, 27, 3, 12, 1, 11, 3, 44, 43, 1, 5, 2, 46, 17, 13, 19, 1, 12, 7, 23, 1, 17, 6, 13, 3, 5, 27, 7, 
-      46, 36, 19, 25, 1, 1, 3, 8, 15, 3, 45, 45, 45, 37, 6, 15, 3, 5, 38, 14, 41, 1, 13, 45, 45, 39, 44, 29, 43, 48, 51, 50, 
-      37, 5, 17, 46, 47, 31, 5, 42, 49, 38, 39, 24, 7, 11, 44, 35, 21, 6, 15, 5, 47, 13, 28, 45, 34, 27, 24, 15, 35, 13, 7, 25, 
-      43, 13, 14, 5, 3, 5, 46, 45, 45, 35, 5, 21, 28, 3, 13, 4, 30, 19, 45, 45, 40, 37, 5, 40, 17, 9, 3, 9, 13, 4, 17, 33, 
-      44, 39, 17, 45, 28, 5, 11, 45, 19, 45, 21, 44, 31, 43, 49, 48, 15, 3, 1, 44, 45, 43, 11, 1, 1, 27, 43, 45, 39, 3, 1, 3, 
-      5, 31, 1, 43, 23, 19, 7, 5, 45, 31, 11, 7, 3, 9, 5, 7, 13, 43, 47, 45, 46, 47, 14, 3, 25, 45, 7, 17, 32, 21, 3, 17, 
-      5, 11, 31, 40, 45, 20, 45, 45, 32, 38, 47, 38, 45, 41, 49, 30, 45, 5, 36, 31, 22, 3, 46, 45, 13, 21, 23, 5, 46, 45, 33, 19, 
-      25, 1, 7, 13, 5, 44, 23, 29, 23, 24, 7, 5, 37, 13, 29, 13, 7, 17, 7, 43, 3, 21, 7, 44, 1, 19, 15, 9, 34, 43, 26, 3, 
-      17, 5, 6, 5, 7, 3, 33, 35, 43, 41, 48, 47, 30, 45, 19, 7, 5, 33, 11, 34, 25, 1, 21, 11, 29, 7, 1, 4, 5, 10, 14, 3, 
-      44, 11, 47, 45, 33, 3, 9, 7, 40, 23, 9, 1, 3, 1, 7, 5, 9, 9, 6, 11, 45, 41, 45, 19, 5, 11, 10, 39, 9, 19, 3, 11, 
-      43, 42, 1, 13, 35, 5, 32, 7, 5, 5, 43, 37, 3, 32, 17, 3, 23, 1, 13, 45, 17, 1, 21, 45, 43, 46, 49, 47, 45, 30, 9, 31, 
-      19, 42, 19, 44, 17, 14, 19, 25, 1, 7, 5, 45, 19, 13, 7, 3, 1, 1, 9, 21, 37, 9, 11, 1, 3, 37, 27, 13, 5, 21, 33, 3, 
-      27, 9, 27, 1, 39, 1, 46, 21, 10, 13, 21, 40, 22, 35, 41, 9, 33, 3, 17, 8, 45, 46, 42, 46, 47, 47, 47, 48, 48, 47, 43, 48, 
-      37, 39, 50, 35, 3, 40, 45, 40, 46, 36, 34, 28, 9, 9, 37, 9, 5, 7, 13, 31, 1, 7, 3, 3, 44, 45, 25, 15, 1, 21, 43, 25, 
-      1, 38, 34, 42, 31, 23, 33, 35, 37, 20, 7, 15, 3, 7, 7, 27, 45, 45, 48, 47, 45, 44, 47, 23, 25, 36, 11, 3, 18, 24, 27, 13, 
-      41, 13, 5, 5, 7, 19, 15, 7, 5, 14, 45, 45, 37, 1, 5, 17, 21, 41, 17, 37, 53, 41, 45, 45, 45, 45, 45, 45, 45, 45, 43, 47, 
-      47, 48, 53, 47, 47, 47, 49, 27, 45, 47, 47, 47, 47, 45, 45, 45, 47, 43, 48, 34, 34, 43, 46, 15, 37, 21, 5, 27, 11, 1, 9, 7, 
-      19, 15, 1, 1, 19, 3, 36, 27, 29, 13, 21, 9, 17, 5, 16, 45, 23, 34, 3, 1, 7, 25, 28, 13, 29, 15, 11, 19, 17, 1, 27, 23, 
-      31, 19, 41, 41, 40, 47, 28, 31, 26, 26, 36, 17, 5, 1, 23, 1, 45, 34, 49, 51, 34, 43, 37, 5, 41, 15, 5, 21, 1, 7, 9, 19, 
-      5, 11, 39, 19, 45, 45, 38, 17, 9, 1, 15, 11, 5, 13, 47, 46, 48, 45, 19, 32, 7, 19, 5, 7, 23, 29, 5, 45, 41, 37, 1, 5, 
-      27, 5, 5, 7, 19, 9, 1, 35, 48, 38, 38, 39, 42, 43, 21, 23, 43, 41, 7, 3, 7, 13, 1, 46, 47, 46, 23, 46, 45, 25, 7, 9, 
-      21, 7, 41, 13, 20, 1, 21, 15, 37, 5, 40, 45, 45, 5, 45, 46, 15, 33, 46, 12, 13, 7, 24, 7, 5, 30, 7, 46, 13, 8, 44, 35, 
-      45, 33, 40, 36, 47, 47, 29, 43, 36, 43, 45, 42, 36, 19, 7, 7, 43, 3, 44, 25, 48, 29, 11, 45, 30, 1, 17, 13, 25, 1, 48, 45, 
-      45, 45, 44, 49, 37, 9, 21, 17, 15, 7, 15, 25, 1, 1, 9, 43, 33, 11, 3, 29, 45, 45, 9, 7, 7, 27, 47, 45, 47, 48, 45, 47, 
-      26, 1, 43, 15, 45, 17, 1, 5, 35, 31, 9, 3, 9, 19, 9, 21, 43, 5, 27, 1, 5, 9, 4, 34, 19, 3, 7, 11, 45, 46, 45, 45, 
-      46, 47, 47, 44, 45, 43, 27, 9, 17, 15, 19, 44, 45, 46, 47, 47, 45, 45, 
-    }
-  }
-};
-
diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp
index 913ebe84a..110b6492a 100644
--- a/tests/unit_tests/hardfork.cpp
+++ b/tests/unit_tests/hardfork.cpp
@@ -69,6 +69,7 @@ public:
   virtual uint64_t get_block_height(const crypto::hash& h) const { return 0; }
   virtual block_header get_block_header(const crypto::hash& h) const { return block_header(); }
   virtual uint64_t get_block_timestamp(const uint64_t& height) const { return 0; }
+  virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const { return {}; }
   virtual uint64_t get_top_block_timestamp() const { return 0; }
   virtual size_t get_block_size(const uint64_t& height) const { return 128; }
   virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; }
@@ -131,6 +132,7 @@ public:
                         , const size_t& block_size
                         , const difficulty_type& cumulative_difficulty
                         , const uint64_t& coins_generated
+                        , uint64_t num_rct_outs
                         , const crypto::hash& blk_hash
                         ) {
     blocks.push_back(blk);
@@ -183,20 +185,20 @@ TEST(major, Only)
   ASSERT_FALSE(hf.add(mkblock(0, 2), 0));
   ASSERT_FALSE(hf.add(mkblock(2, 2), 0));
   ASSERT_TRUE(hf.add(mkblock(1, 2), 0));
-  db.add_block(mkblock(1, 1), 0, 0, 0, crypto::hash());
+  db.add_block(mkblock(1, 1), 0, 0, 0, 0, crypto::hash());
 
   // block height 1, only version 1 is accepted
   ASSERT_FALSE(hf.add(mkblock(0, 2), 1));
   ASSERT_FALSE(hf.add(mkblock(2, 2), 1));
   ASSERT_TRUE(hf.add(mkblock(1, 2), 1));
-  db.add_block(mkblock(1, 1), 0, 0, 0, crypto::hash());
+  db.add_block(mkblock(1, 1), 0, 0, 0, 0, crypto::hash());
 
   // block height 2, only version 2 is accepted
   ASSERT_FALSE(hf.add(mkblock(0, 2), 2));
   ASSERT_FALSE(hf.add(mkblock(1, 2), 2));
   ASSERT_FALSE(hf.add(mkblock(3, 2), 2));
   ASSERT_TRUE(hf.add(mkblock(2, 2), 2));
-  db.add_block(mkblock(2, 1), 0, 0, 0, crypto::hash());
+  db.add_block(mkblock(2, 1), 0, 0, 0, 0, crypto::hash());
 }
 
 TEST(empty_hardforks, Success)
@@ -210,7 +212,7 @@ TEST(empty_hardforks, Success)
   ASSERT_TRUE(hf.get_state(time(NULL) + 3600*24*400) == HardFork::Ready);
 
   for (uint64_t h = 0; h <= 10; ++h) {
-    db.add_block(mkblock(hf, h, 1), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
   ASSERT_EQ(hf.get(0), 1);
@@ -244,14 +246,14 @@ TEST(check_for_height, Success)
   for (uint64_t h = 0; h <= 4; ++h) {
     ASSERT_TRUE(hf.check_for_height(mkblock(1, 1), h));
     ASSERT_FALSE(hf.check_for_height(mkblock(2, 2), h));  // block version is too high
-    db.add_block(mkblock(hf, h, 1), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
   for (uint64_t h = 5; h <= 10; ++h) {
     ASSERT_FALSE(hf.check_for_height(mkblock(1, 1), h));  // block version is too low
     ASSERT_TRUE(hf.check_for_height(mkblock(2, 2), h));
-    db.add_block(mkblock(hf, h, 2), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 }
@@ -268,19 +270,19 @@ TEST(get, next_version)
 
   for (uint64_t h = 0; h <= 4; ++h) {
     ASSERT_EQ(2, hf.get_next_version());
-    db.add_block(mkblock(hf, h, 1), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
   for (uint64_t h = 5; h <= 9; ++h) {
     ASSERT_EQ(4, hf.get_next_version());
-    db.add_block(mkblock(hf, h, 2), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
   for (uint64_t h = 10; h <= 15; ++h) {
     ASSERT_EQ(4, hf.get_next_version());
-    db.add_block(mkblock(hf, h, 4), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, 4), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 }
@@ -321,7 +323,7 @@ TEST(steps_asap, Success)
   hf.init();
 
   for (uint64_t h = 0; h < 10; ++h) {
-    db.add_block(mkblock(hf, h, 9), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, 9), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
@@ -348,7 +350,7 @@ TEST(steps_1, Success)
   hf.init();
 
   for (uint64_t h = 0 ; h < 10; ++h) {
-    db.add_block(mkblock(hf, h, h+1), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, h+1), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
@@ -373,7 +375,7 @@ TEST(reorganize, Same)
     //                                 index  0  1  2  3  4  5  6  7  8  9
     static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
     for (uint64_t h = 0; h < 20; ++h) {
-      db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, crypto::hash());
+      db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash());
       ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
     }
 
@@ -404,7 +406,7 @@ TEST(reorganize, Changed)
   static const uint8_t block_versions[] =    { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
   static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9 };
   for (uint64_t h = 0; h < 16; ++h) {
-    db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE (hf.add(db.get_block_from_height(h), h));
   }
 
@@ -424,7 +426,7 @@ TEST(reorganize, Changed)
   ASSERT_EQ(db.height(), 3);
   hf.reorganize_from_block_height(2);
   for (uint64_t h = 3; h < 16; ++h) {
-    db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, 0, crypto::hash());
     bool ret = hf.add(db.get_block_from_height(h), h);
     ASSERT_EQ (ret, h < 15);
   }
@@ -448,7 +450,7 @@ TEST(voting, threshold)
 
     for (uint64_t h = 0; h <= 8; ++h) {
       uint8_t v = 1 + !!(h % 8);
-      db.add_block(mkblock(hf, h, v), 0, 0, 0, crypto::hash());
+      db.add_block(mkblock(hf, h, v), 0, 0, 0, 0, crypto::hash());
       bool ret = hf.add(db.get_block_from_height(h), h);
       if (h >= 8 && threshold == 87) {
         // for threshold 87, we reach the treshold at height 7, so from height 8, hard fork to version 2, but 8 tries to add 1
@@ -482,7 +484,7 @@ TEST(voting, different_thresholds)
     static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4 };
 
     for (uint64_t h = 0; h < sizeof(block_versions) / sizeof(block_versions[0]); ++h) {
-      db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, crypto::hash());
+      db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash());
       bool ret = hf.add(db.get_block_from_height(h), h);
       ASSERT_EQ(ret, true);
     }
@@ -536,7 +538,7 @@ TEST(voting, info)
     ASSERT_EQ(expected_thresholds[h], threshold);
     ASSERT_EQ(4, voting);
 
-    db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, crypto::hash());
+    db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash());
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 }
@@ -599,7 +601,7 @@ TEST(reorganize, changed)
 #define ADD(v, h, a) \
   do { \
     cryptonote::block b = mkblock(hf, h, v); \
-    db.add_block(b, 0, 0, 0, crypto::hash()); \
+    db.add_block(b, 0, 0, 0, 0, crypto::hash()); \
     ASSERT_##a(hf.add(b, h)); \
   } while(0)
 #define ADD_TRUE(v, h) ADD(v, h, TRUE)