Merge pull request #3981

45e419b db: store cumulative rct output distribution in the db for speed (moneromooo-monero)

8 files changed, 294 insertions, 158 deletions

diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp
index 88ac34255..8544cc3f0 100644
--- a/src/blockchain_db/blockchain_db.cpp
+++ b/src/blockchain_db/blockchain_db.cpp
@@ -218,13 +218,22 @@ uint64_t BlockchainDB::add_block( const block& blk
   // call out to add the transactions
 
   time1 = epee::misc_utils::get_tick_count();
+
+  uint64_t num_rct_outs = 0;
   add_transaction(blk_hash, blk.miner_tx);
+  if (blk.miner_tx.version == 2)
+    num_rct_outs += blk.miner_tx.vout.size();
   int tx_i = 0;
   crypto::hash tx_hash = crypto::null_hash;
   for (const transaction& tx : txs)
   {
     tx_hash = blk.tx_hashes[tx_i];
     add_transaction(blk_hash, tx, &tx_hash);
+    for (const auto &vout: tx.vout)
+    {
+      if (vout.amount == 0)
+        ++num_rct_outs;
+    }
     ++tx_i;
   }
   TIME_MEASURE_FINISH(time1);
@@ -232,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk
 
   // call out to subclass implementation to add the block & metadata
   time1 = epee::misc_utils::get_tick_count();
-  add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash);
+  add_block(blk, block_size, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash);
   TIME_MEASURE_FINISH(time1);
   time_add_block1 += time1;
 
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
index 4648d0e98..d91d3d5fc 100644
--- a/src/blockchain_db/blockchain_db.h
+++ b/src/blockchain_db/blockchain_db.h
@@ -367,6 +367,7 @@ private:
                 , const size_t& block_size
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
+                , uint64_t num_rct_outs
                 , const crypto::hash& blk_hash
                 ) = 0;
 
@@ -906,6 +907,20 @@ public:
   virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0;
 
   /**
+   * @brief fetch a block's cumulative number of rct outputs
+   *
+   * The subclass should return the numer of rct outputs in the blockchain
+   * up to the block with the given height (inclusive).
+   *
+   * If the block does not exist, the subclass should throw BLOCK_DNE
+   *
+   * @param height the height requested
+   *
+   * @return the cumulative number of rct outputs
+   */
+  virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const = 0;
+
+  /**
    * @brief fetch the top block's timestamp
    *
    * The subclass should return the timestamp of the most recent block.
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index b850d5401..b7e0242dc 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -53,7 +53,7 @@ using epee::string_tools::pod_to_hex;
 using namespace crypto;
 
 // Increase when the DB structure changes
-#define VERSION 2
+#define VERSION 3
 
 namespace
 {
@@ -250,6 +250,16 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi
 namespace cryptonote
 {
 
+typedef struct mdb_block_info_old
+{
+  uint64_t bi_height;
+  uint64_t bi_timestamp;
+  uint64_t bi_coins;
+  uint64_t bi_size; // a size_t really but we need 32-bit compat
+  difficulty_type bi_diff;
+  crypto::hash bi_hash;
+} mdb_block_info_old;
+
 typedef struct mdb_block_info
 {
   uint64_t bi_height;
@@ -258,6 +268,7 @@ typedef struct mdb_block_info
   uint64_t bi_size; // a size_t really but we need 32-bit compat
   difficulty_type bi_diff;
   crypto::hash bi_hash;
+  uint64_t bi_cum_rct;
 } mdb_block_info;
 
 typedef struct blk_height {
@@ -667,7 +678,7 @@ estim:
 }
 
 void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
-    const crypto::hash& blk_hash)
+    uint64_t num_rct_outs, const crypto::hash& blk_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -715,6 +726,16 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   bi.bi_size = block_size;
   bi.bi_diff = cumulative_difficulty;
   bi.bi_hash = blk_hash;
+  bi.bi_cum_rct = num_rct_outs;
+  if (blk.major_version >= 4)
+  {
+    uint64_t last_height = m_height-1;
+    MDB_val_set(h, last_height);
+    if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH)))
+        throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str()));
+    const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data;
+    bi.bi_cum_rct += bi_prev->bi_cum_rct;
+  }
 
   MDB_val_set(val, bi);
   result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP);
@@ -759,8 +780,6 @@ void BlockchainLMDB::remove_block()
   if ((result = mdb_cursor_del(m_cur_block_heights, 0)))
       throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str()));
 
-  if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET)))
-      throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str()));
   if ((result = mdb_cursor_del(m_cur_blocks, 0)))
       throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str()));
 
@@ -1143,6 +1162,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB()
   m_cum_size = 0;
   m_cum_count = 0;
 
+  // reset may also need changing when initialize things here
+
   m_hardfork = nullptr;
 }
 
@@ -1923,6 +1944,43 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
   return ret;
 }
 
+std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  std::vector<uint64_t> res;
+  int result;
+
+  if (heights.empty())
+    return {};
+  res.reserve(heights.size());
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(block_info);
+
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_blocks, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+  for (size_t i = 0; i < heights.size(); ++i)
+    if (heights[i] >= db_stats.ms_entries)
+      throw0(BLOCK_DNE(std::string("Attempt to get rct distribution from height " + std::to_string(heights[i]) + " failed -- block size not in db").c_str()));
+
+  MDB_val v;
+
+  for (uint64_t height: heights)
+  {
+    MDB_val_set(v, height);
+    result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str()));
+    const mdb_block_info *bi = (const mdb_block_info *)v.mv_data;
+    res.push_back(bi->bi_cum_rct);
+  }
+
+  TXN_POSTFIX_RDONLY();
+  return res;
+}
+
 uint64_t BlockchainLMDB::get_top_block_timestamp() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -3343,6 +3401,7 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig
   MDB_val_set(k, amount);
   MDB_val v;
   MDB_cursor_op op = MDB_SET;
+  base = 0;
   while (1)
   {
     int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
@@ -3361,6 +3420,9 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig
       break;
   }
 
+  for (size_t n = 1; n < distribution.size(); ++n)
+    distribution[n] += distribution[n - 1];
+
   TXN_POSTFIX_RDONLY();
 
   return true;
@@ -3455,7 +3517,7 @@ void BlockchainLMDB::fixup()
     if (result) \
       throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \
     ptr = (char *)k.mv_data; \
-    ptr[sizeof(name)-2] = 's'
+    ptr[sizeof(name)-2]++
 
 #define LOGIF(y)    if (ELPP->vRegistry()->allowed(y, "global"))
 
@@ -3595,7 +3657,7 @@ void BlockchainLMDB::migrate_0_1()
       break;
     }
     MDB_dbi diffs, hashes, sizes, timestamps;
-    mdb_block_info bi;
+    mdb_block_info_old bi;
     MDB_val_set(nv, bi);
 
     lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs");
@@ -4135,6 +4197,141 @@ void BlockchainLMDB::migrate_1_2()
   txn.commit();
 }
 
+void BlockchainLMDB::migrate_2_3()
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  uint64_t i;
+  int result;
+  mdb_txn_safe txn(false);
+  MDB_val k, v;
+  char *ptr;
+
+  MGINFO_YELLOW("Migrating blockchain from DB version 2 to 3 - this may take a while:");
+
+  do {
+    LOG_PRINT_L1("migrating block info:");
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+
+    MDB_stat db_stats;
+    if ((result = mdb_stat(txn, m_blocks, &db_stats)))
+      throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+    const uint64_t blockchain_height = db_stats.ms_entries;
+
+    MDEBUG("enumerating rct outputs...");
+    std::vector<uint64_t> distribution(blockchain_height, 0);
+    bool r = for_all_outputs(0, [&](uint64_t height) {
+      if (height >= blockchain_height)
+      {
+        MERROR("Output found claiming height >= blockchain height");
+        return false;
+      }
+      distribution[height]++;
+      return true;
+    });
+    if (!r)
+      throw0(DB_ERROR("Failed to build rct output distribution"));
+    for (size_t i = 1; i < distribution.size(); ++i)
+      distribution[i] += distribution[i - 1];
+
+    /* the block_info table name is the same but the old version and new version
+     * have incompatible data. Create a new table. We want the name to be similar
+     * to the old name so that it will occupy the same location in the DB.
+     */
+    MDB_dbi o_block_info = m_block_info;
+    lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+    MDB_cursor *c_old, *c_cur;
+    i = 0;
+    while(1) {
+      if (!(i % 1000)) {
+        if (i) {
+          LOGIF(el::Level::Info) {
+            std::cout << i << " / " << blockchain_height << "  \r" << std::flush;
+          }
+          txn.commit();
+          result = mdb_txn_begin(m_env, NULL, 0, txn);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+        }
+        result = mdb_cursor_open(txn, m_block_info, &c_cur);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str()));
+        result = mdb_cursor_open(txn, o_block_info, &c_old);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
+        if (!i) {
+          MDB_stat db_stat;
+          result = mdb_stat(txn, m_block_info, &db_stats);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str()));
+          i = db_stats.ms_entries;
+        }
+      }
+      result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
+      if (result == MDB_NOTFOUND) {
+        txn.commit();
+        break;
+      }
+      else if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str()));
+      const mdb_block_info_old *bi_old = (const mdb_block_info_old*)v.mv_data;
+      mdb_block_info bi;
+      bi.bi_height = bi_old->bi_height;
+      bi.bi_timestamp = bi_old->bi_timestamp;
+      bi.bi_coins = bi_old->bi_coins;
+      bi.bi_size = bi_old->bi_size;
+      bi.bi_diff = bi_old->bi_diff;
+      bi.bi_hash = bi_old->bi_hash;
+      if (bi_old->bi_height >= distribution.size())
+        throw0(DB_ERROR("Bad height in block_info record"));
+      bi.bi_cum_rct = distribution[bi_old->bi_height];
+      MDB_val_set(nv, bi);
+      result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str()));
+      /* we delete the old records immediately, so the overall DB and mapsize should not grow.
+       * This is a little slower than just letting mdb_drop() delete it all at the end, but
+       * it saves a significant amount of disk space.
+       */
+      result = mdb_cursor_del(c_old, 0);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str()));
+      i++;
+    }
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+    /* Delete the old table */
+    result = mdb_drop(txn, o_block_info, 1);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str()));
+
+    RENAME_DB("block_infn");
+
+    lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+    txn.commit();
+  } while(0);
+
+  uint32_t version = 3;
+  v.mv_data = (void *)&version;
+  v.mv_size = sizeof(version);
+  MDB_val_copy<const char *> vk("version");
+  result = mdb_txn_begin(m_env, NULL, 0, txn);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+  result = mdb_put(txn, m_properties, &vk, &v, 0);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
+  txn.commit();
+}
+
 void BlockchainLMDB::migrate(const uint32_t oldversion)
 {
   switch(oldversion) {
@@ -4142,6 +4339,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion)
     migrate_0_1(); /* FALLTHRU */
   case 1:
     migrate_1_2(); /* FALLTHRU */
+  case 2:
+    migrate_2_3(); /* FALLTHRU */
   default:
     ;
   }
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index 54aa864a6..6210d3687 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -199,6 +199,8 @@ public:
 
   virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const;
 
+  virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const;
+
   virtual uint64_t get_block_timestamp(const uint64_t& height) const;
 
   virtual uint64_t get_top_block_timestamp() const;
@@ -319,6 +321,7 @@ private:
                 , const size_t& block_size
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
+                , uint64_t num_rct_outs
                 , const crypto::hash& block_hash
                 );
 
@@ -389,6 +392,9 @@ private:
   // migrate from DB version 1 to 2
   void migrate_1_2();
 
+  // migrate from DB version 2 to 3
+  void migrate_2_3();
+
   void cleanup_batch();
 
 private:
diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp
index 5d5121504..c729f5cc6 100644
--- a/src/cryptonote_core/blockchain.cpp
+++ b/src/cryptonote_core/blockchain.cpp
@@ -1996,14 +1996,14 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint
 //------------------------------------------------------------------
 bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const
 {
-  // rct outputs don't exist before v3
+  // rct outputs don't exist before v4
   if (amount == 0)
   {
     switch (m_nettype)
     {
-      case STAGENET: start_height = stagenet_hard_forks[2].height; break;
-      case TESTNET: start_height = testnet_hard_forks[2].height; break;
-      case MAINNET: start_height = mainnet_hard_forks[2].height; break;
+      case STAGENET: start_height = stagenet_hard_forks[3].height; break;
+      case TESTNET: start_height = testnet_hard_forks[3].height; break;
+      case MAINNET: start_height = mainnet_hard_forks[3].height; break;
       default: return false;
     }
   }
@@ -2011,11 +2011,40 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height,
     start_height = 0;
   base = 0;
 
+  if (to_height > 0 && to_height < from_height)
+    return false;
+
   const uint64_t real_start_height = start_height;
   if (from_height > start_height)
     start_height = from_height;
 
-  return m_db->get_output_distribution(amount, start_height, to_height, distribution, base);
+  distribution.clear();
+  uint64_t db_height = m_db->height();
+  if (db_height == 0)
+    return false;
+  if (to_height == 0)
+    to_height = db_height - 1;
+  if (start_height >= db_height || to_height >= db_height)
+    return false;
+  if (amount == 0)
+  {
+    std::vector<uint64_t> heights;
+    heights.reserve(to_height + 1 - start_height);
+    uint64_t real_start_height = start_height > 0 ? start_height-1 : start_height;
+    for (uint64_t h = real_start_height; h <= to_height; ++h)
+      heights.push_back(h);
+    distribution = m_db->get_block_cumulative_rct_outputs(heights);
+    if (start_height > 0)
+    {
+      base = distribution[0];
+      distribution.erase(distribution.begin());
+    }
+    return true;
+  }
+  else
+  {
+    return m_db->get_output_distribution(amount, start_height, to_height, distribution, base);
+  }
 }
 //------------------------------------------------------------------
 // This function takes a list of block hashes from another node
diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp
index 029c4ac09..7db6b7f20 100644
--- a/src/rpc/core_rpc_server.cpp
+++ b/src/rpc/core_rpc_server.cpp
@@ -47,7 +47,6 @@ using namespace epee;
 #include "rpc/rpc_args.h"
 #include "core_rpc_server_error_codes.h"
 #include "p2p/net_node.h"
-#include "get_output_distribution_cache.h"
 #include "version.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -2147,6 +2146,10 @@ namespace cryptonote
   bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp)
   {
     PERF_TIMER(on_get_output_distribution);
+    bool r;
+    if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r))
+      return r;
+
     try
     {
       for (uint64_t amount: req.amounts)
@@ -2163,38 +2166,17 @@ namespace cryptonote
 
         if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req.to_height)
         {
-          res.distributions.push_back({amount, d.cached_start_height, d.cached_distribution, d.cached_base});
-          if (req.cumulative)
+          res.distributions.push_back({amount, d.cached_start_height, req.binary, d.cached_distribution, d.cached_base});
+          if (!req.cumulative)
           {
             auto &distribution = res.distributions.back().distribution;
-            distribution[0] += d.cached_base;
-            for (size_t n = 1; n < distribution.size(); ++n)
-              distribution[n] += distribution[n-1];
+            for (size_t n = distribution.size() - 1; n > 0; --n)
+              distribution[n] -= distribution[n-1];
+            distribution[0] -= d.cached_base;
           }
           continue;
         }
 
-        // this is a slow operation, so we have precomputed caches of common cases
-        bool found = false;
-        for (const auto &slot: get_output_distribution_cache)
-        {
-          if (slot.amount == amount && slot.from_height == req.from_height && slot.to_height == req.to_height)
-          {
-            res.distributions.push_back({amount, slot.start_height, slot.distribution, slot.base});
-            found = true;
-            if (req.cumulative)
-            {
-              auto &distribution = res.distributions.back().distribution;
-              distribution[0] += slot.base;
-              for (size_t n = 1; n < distribution.size(); ++n)
-                distribution[n] += distribution[n-1];
-            }
-            break;
-          }
-        }
-        if (found)
-          continue;
-
         std::vector<uint64_t> distribution;
         uint64_t start_height, base;
         if (!m_core.get_output_distribution(amount, req.from_height, req.to_height, start_height, distribution, base))
@@ -2220,14 +2202,14 @@ namespace cryptonote
           d.cached = true;
         }
 
-        if (req.cumulative)
+        if (!req.cumulative)
         {
-          distribution[0] += base;
-          for (size_t n = 1; n < distribution.size(); ++n)
-            distribution[n] += distribution[n-1];
+          for (size_t n = distribution.size() - 1; n > 0; --n)
+            distribution[n] -= distribution[n-1];
+          distribution[0] -= base;
         }
 
-        res.distributions.push_back({amount, start_height, std::move(distribution), base});
+        res.distributions.push_back({amount, start_height, req.binary, std::move(distribution), base});
       }
     }
     catch (const std::exception &e)
diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h
index b05dcdfc0..6738171e2 100644
--- a/src/rpc/core_rpc_server_commands_defs.h
+++ b/src/rpc/core_rpc_server_commands_defs.h
@@ -2262,12 +2262,14 @@ namespace cryptonote
       uint64_t from_height;
       uint64_t to_height;
       bool cumulative;
+      bool binary;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(amounts)
         KV_SERIALIZE_OPT(from_height, (uint64_t)0)
         KV_SERIALIZE_OPT(to_height, (uint64_t)0)
         KV_SERIALIZE_OPT(cumulative, false)
+        KV_SERIALIZE_OPT(binary, true)
       END_KV_SERIALIZE_MAP()
     };
 
@@ -2275,13 +2277,18 @@ namespace cryptonote
     {
       uint64_t amount;
       uint64_t start_height;
+      bool binary;
       std::vector<uint64_t> distribution;
       uint64_t base;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(amount)
         KV_SERIALIZE(start_height)
-        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution)
+        KV_SERIALIZE(binary)
+        if (this_ref.binary)
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution)
+        else
+          KV_SERIALIZE(distribution)
         KV_SERIALIZE(base)
       END_KV_SERIALIZE_MAP()
     };
@@ -2290,10 +2297,12 @@ namespace cryptonote
     {
       std::string status;
       std::vector<distribution> distributions;
+      bool untrusted;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(status)
         KV_SERIALIZE(distributions)
+        KV_SERIALIZE(untrusted)
       END_KV_SERIALIZE_MAP()
     };
   };
diff --git a/src/rpc/get_output_distribution_cache.h b/src/rpc/get_output_distribution_cache.h
deleted file mode 100644
index 6495e7d4c..000000000
--- a/src/rpc/get_output_distribution_cache.h
+++ /dev/null
@@ -1,113 +0,0 @@
-static const struct
-{
-  uint64_t amount;
-  uint64_t from_height;
-  uint64_t to_height;
-  uint64_t start_height;
-  uint64_t base;
-  std::vector<uint64_t> distribution;
-}
-get_output_distribution_cache[] =
-{
-  {
-    0,
-    1544704,
-    1546001,
-    1544704,
-    5143500,
-    {
-      5, 38, 37, 33, 39, 7, 1, 1, 5, 9, 7, 5, 17, 5, 3, 9, 3, 17, 5, 17, 1, 1, 15, 13, 3, 10, 5, 3, 34, 1, 45, 7, 
-      5, 17, 5, 22, 3, 1, 17, 16, 5, 1, 3, 43, 5, 13, 3, 23, 9, 7, 9, 13, 1, 11, 1, 17, 1, 3, 16, 11, 5, 11, 7, 7, 
-      33, 11, 7, 1, 5, 1, 21, 19, 1, 17, 1, 49, 17, 3, 3, 9, 35, 46, 46, 39, 26, 33, 21, 3, 23, 3, 9, 37, 1, 33, 11, 32, 
-      1, 13, 16, 12, 3, 21, 1, 18, 3, 19, 1, 25, 5, 3, 18, 7, 17, 5, 9, 15, 7, 7, 11, 9, 9, 17, 5, 16, 1, 3, 13, 3, 
-      5, 5, 5, 13, 5, 9, 5, 13, 3, 17, 15, 36, 13, 3, 20, 12, 6, 23, 17, 10, 22, 23, 1, 7, 21, 6, 23, 1, 3, 19, 13, 1, 
-      3, 43, 35, 13, 1, 31, 7, 3, 17, 1, 15, 5, 11, 15, 24, 1, 18, 13, 5, 15, 1, 29, 3, 3, 13, 3, 15, 7, 17, 3, 1, 1, 
-      17, 1, 1, 45, 39, 27, 45, 46, 34, 7, 3, 3, 9, 3, 3, 11, 7, 5, 9, 25, 19, 3, 33, 1, 5, 17, 1, 45, 4, 1, 45, 11, 
-      44, 32, 3, 1, 3, 7, 17, 15, 5, 45, 35, 41, 1, 35, 3, 3, 19, 1, 9, 17, 29, 29, 3, 1, 13, 1, 3, 47, 21, 13, 7, 1, 
-      7, 5, 1, 11, 1, 40, 9, 7, 3, 3, 13, 25, 1, 47, 5, 7, 3, 7, 31, 40, 34, 6, 3, 15, 3, 31, 5, 13, 27, 9, 12, 21, 
-      3, 1, 19, 1, 19, 5, 47, 49, 47, 42, 50, 34, 29, 23, 1, 5, 9, 16, 11, 7, 1, 19, 7, 5, 1, 15, 1, 1, 9, 13, 9, 5, 
-      27, 3, 3, 29, 1, 33, 3, 9, 5, 35, 5, 1, 17, 7, 3, 39, 3, 28, 19, 1, 1, 9, 1, 3, 27, 1, 37, 3, 1, 1, 16, 3, 
-      25, 11, 5, 3, 33, 45, 17, 11, 7, 22, 9, 1, 5, 5, 5, 15, 1, 15, 9, 7, 11, 13, 37, 49, 46, 38, 11, 1, 25, 1, 13, 18, 
-      3, 7, 39, 3, 37, 19, 35, 3, 1, 3, 19, 1, 3, 15, 21, 3, 27, 1, 45, 48, 1, 13, 29, 9, 1, 1, 46, 43, 5, 15, 3, 7, 
-      29, 26, 5, 5, 21, 37, 17, 21, 3, 13, 1, 5, 1, 17, 5, 31, 13, 1, 11, 3, 46, 9, 3, 7, 1, 1, 41, 1, 21, 1, 5, 12, 
-      7, 13, 9, 25, 1, 47, 47, 48, 48, 48, 48, 48, 47, 48, 45, 45, 33, 52, 50, 46, 45, 47, 35, 41, 38, 35, 42, 38, 34, 41, 39, 35, 
-      51, 51, 45, 43, 49, 52, 53, 45, 42, 46, 37, 53, 49, 41, 46, 49, 46, 47, 48, 37, 41, 33, 43, 38, 15, 3, 3, 27, 11, 5, 23, 13, 
-      1, 1, 37, 3, 15, 3, 30, 13, 3, 45, 12, 3, 5, 11, 1, 1, 21, 9, 11, 19, 1, 1, 1, 25, 5, 21, 3, 1, 32, 44, 3, 33, 
-      11, 7, 5, 23, 1, 37, 47, 48, 48, 48, 48, 48, 48, 46, 47, 47, 50, 45, 49, 50, 46, 47, 49, 45, 51, 49, 50, 49, 49, 46, 47, 48, 
-      46, 48, 46, 50, 46, 43, 46, 46, 48, 47, 46, 47, 45, 49, 46, 43, 50, 45, 45, 49, 45, 48, 45, 49, 48, 45, 45, 51, 45, 51, 45, 46, 
-      52, 45, 45, 51, 51, 52, 44, 45, 52, 50, 50, 46, 47, 51, 51, 46, 47, 47, 47, 50, 47, 51, 48, 49, 51, 50, 48, 48, 48, 50, 49, 49, 
-      52, 52, 49, 50, 49, 49, 49, 51, 52, 49, 52, 50, 49, 47, 29, 15, 39, 17, 31, 5, 40, 5, 18, 23, 25, 7, 35, 26, 5, 31, 49, 22, 
-      3, 17, 7, 49, 7, 49, 47, 12, 44, 46, 36, 15, 3, 1, 47, 13, 35, 40, 5, 21, 19, 39, 21, 33, 31, 29, 1, 1, 37, 1, 15, 47, 
-      7, 7, 47, 41, 13, 3, 47, 31, 9, 33, 13, 43, 29, 5, 1, 9, 33, 7, 27, 15, 15, 25, 5, 43, 22, 31, 7, 1, 47, 1, 15, 27, 
-      3, 27, 45, 15, 1, 36, 17, 1, 23, 39, 38, 45, 7, 7, 19, 7, 11, 47, 33, 16, 3, 45, 45, 45, 9, 27, 3, 3, 21, 3, 7, 21, 
-      7, 3, 43, 1, 17, 1, 45, 37, 46, 5, 5, 13, 46, 40, 48, 48, 45, 34, 1, 46, 19, 25, 9, 7, 47, 23, 37, 31, 3, 25, 13, 46, 
-      31, 25, 5, 46, 35, 52, 11, 23, 27, 4, 15, 11, 11, 11, 9, 34, 7, 9, 15, 34, 9, 27, 37, 28, 25, 45, 13, 30, 5, 25, 15, 7, 
-      3, 19, 27, 1, 7, 11, 1, 32, 3, 45, 11, 9, 21, 25, 9, 13, 13, 1, 7, 1, 33, 11, 5, 3, 3, 27, 27, 5, 3, 37, 17, 17, 
-      3, 7, 5, 13, 1, 3, 44, 45, 26, 25, 1, 13, 3, 13, 3, 11, 1, 11, 7, 45, 3, 3, 1, 43, 1, 19, 3, 1, 15, 5, 39, 7, 
-      7, 1, 9, 1, 11, 19, 3, 35, 29, 7, 15, 11, 40, 7, 44, 38, 34, 7, 9, 7, 1, 27, 1, 9, 5, 45, 1, 21, 3, 1, 5, 9, 
-      3, 21, 23, 33, 3, 1, 7, 3, 3, 7, 41, 9, 7, 1, 5, 31, 9, 7, 1, 1, 11, 41, 51, 20, 9, 47, 39, 17, 9, 35, 1, 41, 
-      1, 19, 1, 19, 15, 1, 13, 5, 23, 15, 9, 15, 17, 1, 15, 27, 33, 31, 29, 7, 13, 1, 5, 45, 5, 1, 1, 11, 1, 13, 3, 7, 
-      9, 1, 13, 39, 3, 33, 5, 3, 7, 7, 5, 29, 11, 1, 7, 1, 15, 3, 13, 3, 15, 3, 3, 1, 5, 1, 9, 1, 44, 49, 24, 25, 
-      1, 1, 34, 22, 7, 5, 5, 5, 10, 9, 13, 3, 9, 1, 9, 19, 7, 43, 48, 7, 11, 7, 3, 3, 7, 21, 1, 1, 3, 3, 11, 31, 
-      1, 1, 13, 22, 23, 7, 27, 9, 3, 3, 21, 1, 35, 21, 9, 11, 13, 39, 1, 3, 7, 23, 3, 28, 3, 45, 47, 38, 32, 37, 34, 1, 
-      23, 3, 3, 1, 19, 19, 1, 5, 13, 1, 5, 11, 38, 3, 1, 36, 13, 1, 1, 23, 5, 17, 11, 1, 13, 1, 3, 7, 11, 3, 33, 7, 
-      19, 5, 5, 1, 1, 3, 5, 41, 1, 3, 25, 1, 7, 7, 9, 3, 11, 3, 13, 5, 7, 1, 3, 9, 1, 1, 43, 47, 47, 47, 17, 7, 
-      17, 3, 19, 1, 9, 9, 33, 22, 1, 25, 1, 3, 3, 32, 5, 1, 23, 9, 5, 1, 31, 5, 9, 1, 3, 7, 19, 1, 12, 11, 5, 1, 
-      1, 9, 25, 15, 15, 13, 5, 3, 15, 1, 17, 1, 1, 5, 5, 41, 11, 15, 7, 3, 21, 21, 35, 22, 46, 35, 3, 27, 5, 3, 45, 22, 
-      27, 1, 19, 9, 1, 25, 1, 29, 3, 5, 25, 17, 27, 5, 19, 5, 25, 7, 19, 1, 9, 21, 3, 7, 29, 27, 17, 3, 3, 15, 7, 19, 
-      5, 25, 1, 23, 45, 4, 31, 1, 37, 14, 29, 3, 29, 1, 23, 29, 19, 11, 1, 13, 13, 9, 1, 25, 1, 33, 1, 37, 37, 23, 7, 21, 
-      7, 3, 13, 7, 3, 7, 21, 11, 9, 1, 31, 3, 1, 7, 39, 46, 3, 30, 
-    },
-  },
-  {
-    0,
-    1562704,
-    1564001,
-    1562704,
-    5521986,
-    {
-      35, 45, 23, 3, 44, 47, 44, 3, 17, 35, 7, 11, 7, 29, 43, 27, 11, 7, 5, 31, 13, 9, 45, 45, 7, 42, 17, 15, 19, 11, 45, 19, 
-      45, 46, 45, 46, 32, 34, 43, 34, 46, 47, 45, 30, 17, 45, 46, 36, 35, 38, 19, 9, 23, 17, 3, 19, 31, 41, 35, 24, 15, 45, 15, 5, 
-      11, 5, 19, 11, 11, 7, 15, 19, 45, 34, 7, 7, 29, 1, 9, 36, 7, 44, 45, 33, 25, 8, 17, 7, 44, 43, 48, 45, 42, 46, 40, 44, 
-      1, 43, 45, 46, 46, 35, 19, 19, 23, 5, 13, 19, 7, 16, 9, 3, 25, 34, 3, 27, 9, 39, 3, 43, 21, 1, 45, 45, 39, 25, 23, 13, 
-      39, 39, 3, 45, 43, 46, 44, 40, 39, 33, 45, 47, 38, 45, 45, 39, 47, 47, 45, 46, 35, 45, 43, 47, 45, 40, 34, 42, 42, 48, 49, 47, 
-      47, 48, 47, 45, 43, 48, 37, 48, 41, 45, 48, 34, 42, 44, 9, 19, 27, 1, 47, 47, 43, 25, 29, 5, 5, 21, 39, 35, 43, 37, 13, 45, 
-      25, 31, 26, 47, 45, 23, 23, 39, 32, 25, 44, 47, 35, 47, 15, 17, 7, 9, 5, 35, 31, 3, 45, 47, 46, 13, 17, 48, 45, 9, 13, 45, 
-      45, 31, 1, 53, 44, 46, 39, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 47, 47, 47, 47, 47, 47, 47, 53, 49, 45, 45, 50, 
-      50, 27, 43, 46, 47, 46, 45, 48, 36, 42, 42, 46, 45, 47, 41, 45, 43, 47, 38, 48, 47, 33, 11, 45, 46, 34, 42, 32, 3, 45, 37, 45, 
-      15, 3, 45, 29, 31, 9, 5, 3, 27, 5, 21, 25, 7, 15, 46, 34, 5, 3, 17, 3, 9, 13, 7, 11, 3, 1, 34, 13, 7, 45, 33, 26, 
-      9, 5, 9, 41, 43, 34, 3, 35, 3, 17, 37, 11, 17, 3, 15, 27, 15, 45, 46, 13, 26, 16, 11, 7, 5, 45, 38, 45, 45, 22, 44, 44, 
-      43, 6, 11, 35, 15, 44, 17, 27, 13, 3, 40, 5, 9, 7, 35, 19, 5, 5, 1, 28, 33, 15, 45, 5, 29, 3, 31, 12, 5, 32, 24, 3, 
-      23, 13, 47, 45, 42, 46, 39, 21, 21, 1, 44, 44, 47, 41, 5, 1, 11, 36, 20, 5, 45, 39, 45, 45, 44, 45, 32, 22, 40, 11, 38, 1, 
-      45, 46, 37, 9, 23, 9, 15, 44, 7, 16, 38, 46, 11, 14, 24, 19, 19, 7, 26, 25, 45, 37, 17, 1, 35, 1, 3, 28, 3, 11, 22, 13, 
-      3, 1, 7, 38, 5, 3, 1, 26, 1, 3, 43, 44, 46, 45, 21, 11, 1, 44, 27, 1, 11, 26, 10, 44, 45, 45, 45, 47, 47, 45, 48, 45, 
-      38, 9, 5, 44, 46, 27, 3, 12, 1, 11, 3, 44, 43, 1, 5, 2, 46, 17, 13, 19, 1, 12, 7, 23, 1, 17, 6, 13, 3, 5, 27, 7, 
-      46, 36, 19, 25, 1, 1, 3, 8, 15, 3, 45, 45, 45, 37, 6, 15, 3, 5, 38, 14, 41, 1, 13, 45, 45, 39, 44, 29, 43, 48, 51, 50, 
-      37, 5, 17, 46, 47, 31, 5, 42, 49, 38, 39, 24, 7, 11, 44, 35, 21, 6, 15, 5, 47, 13, 28, 45, 34, 27, 24, 15, 35, 13, 7, 25, 
-      43, 13, 14, 5, 3, 5, 46, 45, 45, 35, 5, 21, 28, 3, 13, 4, 30, 19, 45, 45, 40, 37, 5, 40, 17, 9, 3, 9, 13, 4, 17, 33, 
-      44, 39, 17, 45, 28, 5, 11, 45, 19, 45, 21, 44, 31, 43, 49, 48, 15, 3, 1, 44, 45, 43, 11, 1, 1, 27, 43, 45, 39, 3, 1, 3, 
-      5, 31, 1, 43, 23, 19, 7, 5, 45, 31, 11, 7, 3, 9, 5, 7, 13, 43, 47, 45, 46, 47, 14, 3, 25, 45, 7, 17, 32, 21, 3, 17, 
-      5, 11, 31, 40, 45, 20, 45, 45, 32, 38, 47, 38, 45, 41, 49, 30, 45, 5, 36, 31, 22, 3, 46, 45, 13, 21, 23, 5, 46, 45, 33, 19, 
-      25, 1, 7, 13, 5, 44, 23, 29, 23, 24, 7, 5, 37, 13, 29, 13, 7, 17, 7, 43, 3, 21, 7, 44, 1, 19, 15, 9, 34, 43, 26, 3, 
-      17, 5, 6, 5, 7, 3, 33, 35, 43, 41, 48, 47, 30, 45, 19, 7, 5, 33, 11, 34, 25, 1, 21, 11, 29, 7, 1, 4, 5, 10, 14, 3, 
-      44, 11, 47, 45, 33, 3, 9, 7, 40, 23, 9, 1, 3, 1, 7, 5, 9, 9, 6, 11, 45, 41, 45, 19, 5, 11, 10, 39, 9, 19, 3, 11, 
-      43, 42, 1, 13, 35, 5, 32, 7, 5, 5, 43, 37, 3, 32, 17, 3, 23, 1, 13, 45, 17, 1, 21, 45, 43, 46, 49, 47, 45, 30, 9, 31, 
-      19, 42, 19, 44, 17, 14, 19, 25, 1, 7, 5, 45, 19, 13, 7, 3, 1, 1, 9, 21, 37, 9, 11, 1, 3, 37, 27, 13, 5, 21, 33, 3, 
-      27, 9, 27, 1, 39, 1, 46, 21, 10, 13, 21, 40, 22, 35, 41, 9, 33, 3, 17, 8, 45, 46, 42, 46, 47, 47, 47, 48, 48, 47, 43, 48, 
-      37, 39, 50, 35, 3, 40, 45, 40, 46, 36, 34, 28, 9, 9, 37, 9, 5, 7, 13, 31, 1, 7, 3, 3, 44, 45, 25, 15, 1, 21, 43, 25, 
-      1, 38, 34, 42, 31, 23, 33, 35, 37, 20, 7, 15, 3, 7, 7, 27, 45, 45, 48, 47, 45, 44, 47, 23, 25, 36, 11, 3, 18, 24, 27, 13, 
-      41, 13, 5, 5, 7, 19, 15, 7, 5, 14, 45, 45, 37, 1, 5, 17, 21, 41, 17, 37, 53, 41, 45, 45, 45, 45, 45, 45, 45, 45, 43, 47, 
-      47, 48, 53, 47, 47, 47, 49, 27, 45, 47, 47, 47, 47, 45, 45, 45, 47, 43, 48, 34, 34, 43, 46, 15, 37, 21, 5, 27, 11, 1, 9, 7, 
-      19, 15, 1, 1, 19, 3, 36, 27, 29, 13, 21, 9, 17, 5, 16, 45, 23, 34, 3, 1, 7, 25, 28, 13, 29, 15, 11, 19, 17, 1, 27, 23, 
-      31, 19, 41, 41, 40, 47, 28, 31, 26, 26, 36, 17, 5, 1, 23, 1, 45, 34, 49, 51, 34, 43, 37, 5, 41, 15, 5, 21, 1, 7, 9, 19, 
-      5, 11, 39, 19, 45, 45, 38, 17, 9, 1, 15, 11, 5, 13, 47, 46, 48, 45, 19, 32, 7, 19, 5, 7, 23, 29, 5, 45, 41, 37, 1, 5, 
-      27, 5, 5, 7, 19, 9, 1, 35, 48, 38, 38, 39, 42, 43, 21, 23, 43, 41, 7, 3, 7, 13, 1, 46, 47, 46, 23, 46, 45, 25, 7, 9, 
-      21, 7, 41, 13, 20, 1, 21, 15, 37, 5, 40, 45, 45, 5, 45, 46, 15, 33, 46, 12, 13, 7, 24, 7, 5, 30, 7, 46, 13, 8, 44, 35, 
-      45, 33, 40, 36, 47, 47, 29, 43, 36, 43, 45, 42, 36, 19, 7, 7, 43, 3, 44, 25, 48, 29, 11, 45, 30, 1, 17, 13, 25, 1, 48, 45, 
-      45, 45, 44, 49, 37, 9, 21, 17, 15, 7, 15, 25, 1, 1, 9, 43, 33, 11, 3, 29, 45, 45, 9, 7, 7, 27, 47, 45, 47, 48, 45, 47, 
-      26, 1, 43, 15, 45, 17, 1, 5, 35, 31, 9, 3, 9, 19, 9, 21, 43, 5, 27, 1, 5, 9, 4, 34, 19, 3, 7, 11, 45, 46, 45, 45, 
-      46, 47, 47, 44, 45, 43, 27, 9, 17, 15, 19, 44, 45, 46, 47, 47, 45, 45, 
-    }
-  }
-};
-