Merge pull request #4219

9137ad2c blockchain: add a testnet v9 a day after v8 (moneromooo-monero)
ac4f71c2 wallet2: bump testnet rollback to account for coming reorg (moneromooo-monero)
8f418a6d bulletproofs: #include <openssl/bn.h> (moneromooo-monero)
2bf63650 bulletproofs: speed up the latest changes a bit (moneromooo-monero)
044dff5a bulletproofs: scale points by 8 to ensure subgroup validity (moneromooo-monero)
c83012c4 bulletproofs: match aggregated verification to sarang's latest prototype (moneromooo-monero)
ce0c7432 performance_tests: add padded bulletproof construction (moneromooo-monero)
1224e53b core_tests: add a test for 4-aggregated BP verification (moneromooo-monero)
0e6ed559 fuzz_tests: add a bulletproof fuzz test (moneromooo-monero)
463434d1 more comprehensive test for ge_p3 comparison to identity/point at infinity (moneromooo-monero)
d0a0565f unit_tests: add a few more multiexp unit tests (moneromooo-monero)
6526d87f core_tests: add a test for a tx with empty bulletproof (moneromooo-monero)
a129bbd9 multiexp: fix maxscalar off by one (moneromooo-monero)
7ed496cc ringct: error out when hashToPoint* returns the point at infinity (moneromooo-monero)
d1591853 cryptonote_basic: check output type before using it (moneromooo-monero)
61632dc1 ringct: prevent a potential very large allocation (moneromooo-monero)
a4317e61 crypto: some paranoid checks in generate_signature/check_signature (moneromooo-monero)
7434df1c crypto: never return zero in random32_unbiased (moneromooo-monero)
0825e974 multiexp: fix wrong Bos-Coster result for 1 non trivial input (moneromooo-monero)
a1359ad4 Check inputs to addKeys are in range (moneromooo-monero)
fe0fa3b9 bulletproofs: reject x, y, z, or w[i] being zero (moneromooo-monero)
5ffb2ff9 v8: per byte fee, pad bulletproofs, fixed 11 ring size (moneromooo-monero)
869b3bf8 bulletproofs: a few fixes from the Kudelski review (moneromooo-monero)
c4291762 bulletproofs: reject points not in the main subgroup (moneromooo-monero)
15697177 bulletproofs: speed up a few multiplies using existing Hi cache (moneromooo-monero)
0b05a0fa Add Pippenger cache and limit Straus cache size (moneromooo-monero)
51eb3bdc add pippenger unit tests (moneromooo-monero)
b17b8db3 performance_tests: add stats and loop count multiplier options (moneromooo-monero)
7314d919 perf_timer: split timer class into a base one and a logging one (moneromooo-monero)
d126a02b performance_tests: add aggregated bulletproof tx verification (moneromooo-monero)
263431c4 Pippenger multiexp (moneromooo-monero)
1ed0ed4d multiexp: cut down on memory allocations (moneromooo-monero)
1b867e7f precalc the ge_p3 representation of H (moneromooo-monero)
ef56529f performance_tests: document the tested bulletproof layouts (moneromooo-monero)
30111780 unit_tests: a couple more bulletproof unit tests for gamma (moneromooo-monero)
c444b1b2 require canonical multi output bulletproof layout (moneromooo-monero)
7e67c52f Add a define for the max number of bulletproof multi-outputs (moneromooo-monero)
2a8fcb42 Bulletproof aggregated verification and tests (moneromooo-monero)
126196b0 multiexp: some speedups (moneromooo-monero)
71d67bda aligned: aligned memory alloc/realloc/free (moneromooo-monero)
cb9ecab1 performance_tests: add signature generation/verification (moneromooo-monero)
bacf0a1e bulletproofs: add aggregated verification (moneromooo-monero)
e895c3de make straus cached mode thread safe, and add tests for it (moneromooo-monero)
7f48bf05 multiexp: bos coster now works for just one point (moneromooo-monero)
9ce9f8ca bulletproofs: add multi output bulletproofs to rct (moneromooo-monero)
f34e2e20 performance_tests: add tx checking tests with more than 2 outputs (moneromooo-monero)
0793184b performance_tests: add a --verbose flag, and default to terse (moneromooo-monero)
939bc223 add Straus multiexp (moneromooo-monero)
9ff6e6a0 ringct: add bos coster multiexp (moneromooo-monero)
e9164bb3 bulletproofs: misc optimizations (moneromooo-monero)
112f32f0 performance_tests: add crypto ops (moneromooo-monero)
f5d7b993 performance_tests: add bulletproofs (moneromooo-monero)
8f4ce989 performance_tests: add RingCT MLSAG gen/ver tests (moneromooo-monero)
1aa10c43 performance_tests: add (Borromean) range proofs (moneromooo-monero)
aacfd6e3 bulletproofs: multi-output bulletproofs (moneromooo-monero)
cb1cc757 performance_tests: don't override log level to 0 (moneromooo-monero)

2 files changed, 22 insertions, 17 deletions

diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index d854cbeaf..9e22e2e4b 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -256,7 +256,7 @@ typedef struct mdb_block_info_old
   uint64_t bi_height;
   uint64_t bi_timestamp;
   uint64_t bi_coins;
-  uint64_t bi_size; // a size_t really but we need 32-bit compat
+  uint64_t bi_weight; // a size_t really but we need 32-bit compat
   difficulty_type bi_diff;
   crypto::hash bi_hash;
 } mdb_block_info_old;
@@ -266,7 +266,7 @@ typedef struct mdb_block_info
   uint64_t bi_height;
   uint64_t bi_timestamp;
   uint64_t bi_coins;
-  uint64_t bi_size; // a size_t really but we need 32-bit compat
+  uint64_t bi_weight; // a size_t really but we need 32-bit compat
   difficulty_type bi_diff;
   crypto::hash bi_hash;
   uint64_t bi_cum_rct;
@@ -652,8 +652,11 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin
     block_rtxn_start(&rtxn, &rcurs);
     for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
     {
-      uint32_t block_size = get_block_size(block_num);
-      total_block_size += block_size;
+      // we have access to block weight, which will be greater or equal to block size,
+      // so use this as a proxy. If it's too much off, we might have to check actual size,
+      // which involves reading more data, so is not really wanted
+      size_t block_weight = get_block_weight(block_num);
+      total_block_size += block_weight;
       // Track number of blocks being totalled here instead of assuming, in case
       // some blocks were to be skipped for being outliers.
       ++num_blocks_used;
@@ -674,7 +677,7 @@ estim:
   return threshold_size;
 }
 
-void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
+void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
     uint64_t num_rct_outs, const crypto::hash& blk_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -720,7 +723,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   bi.bi_height = m_height;
   bi.bi_timestamp = blk.timestamp;
   bi.bi_coins = coins_generated;
-  bi.bi_size = block_size;
+  bi.bi_weight = block_weight;
   bi.bi_diff = cumulative_difficulty;
   bi.bi_hash = blk_hash;
   bi.bi_cum_rct = num_rct_outs;
@@ -743,7 +746,9 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   if (result)
     throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str()));
 
-  m_cum_size += block_size;
+  // we use weight as a proxy for size, since we don't have size but weight is >= size
+  // and often actually equal
+  m_cum_size += block_weight;
   m_cum_count++;
 }
 
@@ -2011,7 +2016,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const
   return get_block_timestamp(m_height - 1);
 }
 
-size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
+size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2029,7 +2034,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
     throw0(DB_ERROR("Error attempting to retrieve a block size from the db"));
 
   mdb_block_info *bi = (mdb_block_info *)result.mv_data;
-  size_t ret = bi->bi_size;
+  size_t ret = bi->bi_weight;
   TXN_POSTFIX_RDONLY();
   return ret;
 }
@@ -3090,7 +3095,7 @@ void BlockchainLMDB::block_txn_abort()
   }
 }
 
-uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
+uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
     const std::vector<transaction>& txs)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -3109,7 +3114,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
 
   try
   {
-    BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
+    BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs);
   }
   catch (const DB_ERROR_TXN_START& e)
   {
@@ -3698,9 +3703,9 @@ void BlockchainLMDB::migrate_0_1()
       if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str()));
       if (v.mv_size == sizeof(uint32_t))
-        bi.bi_size = *(uint32_t *)v.mv_data;
+        bi.bi_weight = *(uint32_t *)v.mv_data;
       else
-        bi.bi_size = *(uint64_t *)v.mv_data;  // this is a 32/64 compat bug in version 0
+        bi.bi_weight = *(uint64_t *)v.mv_data;  // this is a 32/64 compat bug in version 0
       result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT);
       if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str()));
@@ -4259,7 +4264,7 @@ void BlockchainLMDB::migrate_2_3()
       bi.bi_height = bi_old->bi_height;
       bi.bi_timestamp = bi_old->bi_timestamp;
       bi.bi_coins = bi_old->bi_coins;
-      bi.bi_size = bi_old->bi_size;
+      bi.bi_weight = bi_old->bi_weight;
       bi.bi_diff = bi_old->bi_diff;
       bi.bi_hash = bi_old->bi_hash;
       if (bi_old->bi_height >= distribution.size())
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index f700d3178..e1f748ed8 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -205,7 +205,7 @@ public:
 
   virtual uint64_t get_top_block_timestamp() const;
 
-  virtual size_t get_block_size(const uint64_t& height) const;
+  virtual size_t get_block_weight(const uint64_t& height) const;
 
   virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const;
 
@@ -273,7 +273,7 @@ public:
   virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const;
 
   virtual uint64_t add_block( const block& blk
-                            , const size_t& block_size
+                            , size_t block_weight
                             , const difficulty_type& cumulative_difficulty
                             , const uint64_t& coins_generated
                             , const std::vector<transaction>& txs
@@ -317,7 +317,7 @@ private:
   uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const;
 
   virtual void add_block( const block& blk
-                , const size_t& block_size
+                , size_t block_weight
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs