Merge pull request #5100

c4851024 wallet_rpc_server: avoid repeated string allocations when parsing (moneromooo-monero)
88c85c18 cryptonote: avoid double parsing blocks when syncing (moneromooo-monero)
9feda0ee cryptonote: speed up calculating coinbase tx prunable hash (moneromooo-monero)
238401d4 core: avoid double parsing blocks after hoh (moneromooo-monero)
dc5a7609 blockchain: avoid unneeded block copy (moneromooo-monero)
79b4e9f3 save some database calls when getting top block hash and height (moneromooo-monero)
98278808 blockchain: avoid pointless transaction copy and temporary (moneromooo-monero)
07d655e4 blockchain: avoid duplicate block hash computation (moneromooo-monero)
f75d51ab core: avoid calculating tx prefix hash when we don't need it (moneromooo-monero)
b044d03a Avoid repeated (de)serialization when syncing (moneromooo-monero)
b747e836 wallet2: don't calculate prefix hash when we don't need it (moneromooo-monero)
e69477bf db: speedup block addition (moneromooo-monero)

3 files changed, 34 insertions, 32 deletions

diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp
index 1e764c83e..ccfcfc15a 100644
--- a/tests/unit_tests/ban.cpp
+++ b/tests/unit_tests/ban.cpp
@@ -56,7 +56,7 @@ public:
   void get_blockchain_top(uint64_t& height, crypto::hash& top_id)const{height=0;top_id=crypto::null_hash;}
   bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; }
   bool handle_incoming_txs(const std::vector<cryptonote::blobdata>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; }
-  bool handle_incoming_block(const cryptonote::blobdata& block_blob, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true) { return true; }
+  bool handle_incoming_block(const cryptonote::blobdata& block_blob, const cryptonote::block *block, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true) { return true; }
   void pause_mine(){}
   void resume_mine(){}
   bool on_idle(){return true;}
@@ -65,7 +65,7 @@ public:
   cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class test_core."); }
   bool get_test_drop_download() const {return true;}
   bool get_test_drop_download_height() const {return true;}
-  bool prepare_handle_incoming_blocks(const std::vector<cryptonote::block_complete_entry>  &blocks) { return true; }
+  bool prepare_handle_incoming_blocks(const std::vector<cryptonote::block_complete_entry>  &blocks_entry, std::vector<cryptonote::block> &blocks) { return true; }
   bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; }
   uint64_t get_target_blockchain_height() const { return 1; }
   size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
diff --git a/tests/unit_tests/blockchain_db.cpp b/tests/unit_tests/blockchain_db.cpp
index 9cf8c5fbe..f9a6adf03 100644
--- a/tests/unit_tests/blockchain_db.cpp
+++ b/tests/unit_tests/blockchain_db.cpp
@@ -163,17 +163,17 @@ protected:
       block bl;
       blobdata bd = h2b(i);
       parse_and_validate_block_from_blob(bd, bl);
-      m_blocks.push_back(bl);
+      m_blocks.push_back(std::make_pair(bl, bd));
     }
     for (auto& i : t_transactions)
     {
-      std::vector<transaction> txs;
+      std::vector<std::pair<transaction, blobdata>> txs;
       for (auto& j : i)
       {
         transaction tx;
         blobdata bd = h2b(j);
         parse_and_validate_tx_from_blob(bd, tx);
-        txs.push_back(tx);
+        txs.push_back(std::make_pair(tx, bd));
       }
       m_txs.push_back(txs);
     }
@@ -187,8 +187,8 @@ protected:
   BlockchainDB* m_db;
   HardFork m_hardfork;
   std::string m_prefix;
-  std::vector<block> m_blocks;
-  std::vector<std::vector<transaction> > m_txs;
+  std::vector<std::pair<block, blobdata>> m_blocks;
+  std::vector<std::vector<std::pair<transaction, blobdata>>> m_txs;
   std::vector<std::string> m_filenames;
 
   void init_hard_fork()
@@ -283,19 +283,19 @@ TYPED_TEST(BlockchainDBTest, AddBlock)
   ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1]));
 
   block b;
-  ASSERT_TRUE(this->m_db->block_exists(get_block_hash(this->m_blocks[0])));
-  ASSERT_NO_THROW(b = this->m_db->get_block(get_block_hash(this->m_blocks[0])));
+  ASSERT_TRUE(this->m_db->block_exists(get_block_hash(this->m_blocks[0].first)));
+  ASSERT_NO_THROW(b = this->m_db->get_block(get_block_hash(this->m_blocks[0].first)));
 
-  ASSERT_TRUE(compare_blocks(this->m_blocks[0], b));
+  ASSERT_TRUE(compare_blocks(this->m_blocks[0].first, b));
 
   ASSERT_NO_THROW(b = this->m_db->get_block_from_height(0));
 
-  ASSERT_TRUE(compare_blocks(this->m_blocks[0], b));
+  ASSERT_TRUE(compare_blocks(this->m_blocks[0].first, b));
 
   // assert that we can't add the same block twice
   ASSERT_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]), TX_EXISTS);
 
-  for (auto& h : this->m_blocks[0].tx_hashes)
+  for (auto& h : this->m_blocks[0].first.tx_hashes)
   {
     transaction tx;
     ASSERT_TRUE(this->m_db->tx_exists(h));
@@ -327,21 +327,21 @@ TYPED_TEST(BlockchainDBTest, RetrieveBlockData)
   ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1]));
   ASSERT_EQ(t_diffs[1] - t_diffs[0], this->m_db->get_block_difficulty(1));
 
-  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), this->m_db->get_block_hash_from_height(0));
+  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0].first), this->m_db->get_block_hash_from_height(0));
 
   std::vector<block> blks;
   ASSERT_NO_THROW(blks = this->m_db->get_blocks_range(0, 1));
   ASSERT_EQ(2, blks.size());
   
-  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), get_block_hash(blks[0]));
-  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1]), get_block_hash(blks[1]));
+  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0].first), get_block_hash(blks[0]));
+  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1].first), get_block_hash(blks[1]));
 
   std::vector<crypto::hash> hashes;
   ASSERT_NO_THROW(hashes = this->m_db->get_hashes_range(0, 1));
   ASSERT_EQ(2, hashes.size());
 
-  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), hashes[0]);
-  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1]), hashes[1]);
+  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0].first), hashes[0]);
+  ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1].first), hashes[1]);
 }
 
 }  // anonymous namespace
diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp
index 0ee6c5c2b..f37b608b6 100644
--- a/tests/unit_tests/long_term_block_weight.cpp
+++ b/tests/unit_tests/long_term_block_weight.cpp
@@ -64,11 +64,13 @@ public:
   virtual uint64_t height() const override { return blocks.size(); }
   virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; }
   virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; }
-  virtual crypto::hash top_block_hash() const override {
+  virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override {
     uint64_t h = height();
     crypto::hash top = crypto::null_hash;
     if (h)
       *(uint64_t*)&top = h - 1;
+    if (block_height)
+      *block_height = h - 1;
     return top;
   }
   virtual void pop_block(cryptonote::block &blk, std::vector<cryptonote::transaction> &txs) override { blocks.pop_back(); }
@@ -123,7 +125,7 @@ TEST(long_term_block_weight, identical_before_fork)
   {
     size_t w = h < CRYPTONOTE_REWARD_BLOCKS_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
   }
   for (uint64_t h = 0; h < 10 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h)
@@ -140,7 +142,7 @@ TEST(long_term_block_weight, identical_after_fork_before_long_term_window)
   {
     size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
   }
   for (uint64_t h = 0; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h)
@@ -157,7 +159,7 @@ TEST(long_term_block_weight, ceiling_at_30000000)
   {
     size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
   }
   ASSERT_EQ(bc->get_current_cumulative_block_weight_median(), 15000000);
@@ -172,7 +174,7 @@ TEST(long_term_block_weight, multi_pop)
   {
     size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
   }
 
@@ -183,7 +185,7 @@ TEST(long_term_block_weight, multi_pop)
   {
     size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
   }
 
@@ -207,7 +209,7 @@ TEST(long_term_block_weight, multiple_updates)
   {
     size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
     const uint64_t effective_median = bc->get_current_cumulative_block_weight_median();
     const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit();
@@ -231,7 +233,7 @@ TEST(long_term_block_weight, pop_invariant_max)
   {
     size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
   }
 
@@ -259,7 +261,7 @@ TEST(long_term_block_weight, pop_invariant_max)
     {
       size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
       uint64_t ltw = bc->get_next_long_term_block_weight(w);
-      bc->get_db().add_block(cryptonote::block(), w, ltw, bc->get_db().height(), bc->get_db().height(), {});
+      bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, bc->get_db().height(), bc->get_db().height(), {});
       ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
     }
 
@@ -279,7 +281,7 @@ TEST(long_term_block_weight, pop_invariant_random)
   {
     size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
   }
 
@@ -314,7 +316,7 @@ TEST(long_term_block_weight, pop_invariant_random)
       uint32_t r = lcg();
       size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : (r % bc->get_current_cumulative_block_weight_limit());
       uint64_t ltw = bc->get_next_long_term_block_weight(w);
-      bc->get_db().add_block(cryptonote::block(), w, ltw, bc->get_db().height(), bc->get_db().height(), {});
+      bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, bc->get_db().height(), bc->get_db().height(), {});
       ASSERT_TRUE(bc->update_next_cumulative_weight_limit());
       const uint64_t effective_median = bc->get_current_cumulative_block_weight_median();
       const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit();
@@ -342,7 +344,7 @@ TEST(long_term_block_weight, long_growth_spike_and_drop)
   {
     size_t w = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5;
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight));
   }
   ASSERT_EQ(long_term_effective_median_block_weight, 300000);
@@ -354,7 +356,7 @@ TEST(long_term_block_weight, long_growth_spike_and_drop)
     float t = h / float(365 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000);
     size_t w = 300000 + t * 30000;
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight));
   }
   ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07);
@@ -365,7 +367,7 @@ TEST(long_term_block_weight, long_growth_spike_and_drop)
   {
     size_t w = bc->get_current_cumulative_block_weight_limit();
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight));
   }
   ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07);
@@ -376,7 +378,7 @@ TEST(long_term_block_weight, long_growth_spike_and_drop)
   {
     size_t w = bc->get_current_cumulative_block_weight_median() * .25;
     uint64_t ltw = bc->get_next_long_term_block_weight(w);
-    bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {});
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
     ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight));
   }
   ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07);