From e69477bf25630c94a5aff666433f3da9ab29298c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 11 Nov 2018 13:39:00 +0000 Subject: db: speedup block addition by avoiding repeated (de)serialization --- src/blockchain_db/lmdb/db_lmdb.cpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'src/blockchain_db') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 9d2f7821e..376170a6b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -865,21 +865,26 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons cryptonote::blobdata blob = tx_to_blob(tx); MDB_val_sized(blobval, blob); - std::stringstream ss; - binary_archive ba(ss); - bool r = const_cast(tx).serialize_base(ba); - if (!r) - throw0(DB_ERROR("Failed to serialize pruned tx")); - std::string pruned = ss.str(); - MDB_val_sized(pruned_blob, pruned); + unsigned int unprunable_size = tx.unprunable_size; + if (unprunable_size == 0) + { + std::stringstream ss; + binary_archive ba(ss); + bool r = const_cast(tx).serialize_base(ba); + if (!r) + throw0(DB_ERROR("Failed to serialize pruned tx")); + unprunable_size = ss.str().size(); + } + + if (unprunable_size > blob.size()) + throw0(DB_ERROR("pruned tx size is larger than tx size")); + + MDB_val pruned_blob = {unprunable_size, (void*)blob.data()}; result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); - if (pruned.size() > blob.size()) - throw0(DB_ERROR("pruned tx size is larger than tx size")); - cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size()); - MDB_val_sized(prunable_blob, prunable); + MDB_val prunable_blob = {blob.size() - unprunable_size, (void*)(blob.data() + unprunable_size)}; result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); -- cgit v1.2.3 From b044d03a51e1dc64bebe2461813e0cfc50f71b0d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 11 Nov 2018 14:51:03 +0000 Subject: Avoid repeated (de)serialization when syncing --- src/blockchain_db/blockchain_db.cpp | 20 ++++++++++++-------- src/blockchain_db/blockchain_db.h | 8 ++++---- src/blockchain_db/lmdb/db_lmdb.cpp | 13 +++++++------ src/blockchain_db/lmdb/db_lmdb.h | 6 +++--- src/blockchain_db/testdb.h | 2 +- 5 files changed, 27 insertions(+), 22 deletions(-) (limited to 'src/blockchain_db') diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 041759593..754c3c2da 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -121,8 +121,10 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) { + const transaction &tx = txp.first; + bool miner_tx = false; crypto::hash tx_hash, tx_prunable_hash; if (!tx_hash_ptr) @@ -138,7 +140,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti if (tx.version >= 2) { if (!tx_prunable_hash_ptr) - tx_prunable_hash = get_transaction_prunable_hash(tx); + tx_prunable_hash = get_transaction_prunable_hash(tx, &txp.second); else tx_prunable_hash = *tx_prunable_hash_ptr; } @@ -168,7 +170,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash, tx_prunable_hash); + uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash); std::vector amount_output_indices(tx.vout.size()); @@ -195,14 +197,16 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti add_tx_amount_output_indices(tx_id, amount_output_indices); } -uint64_t BlockchainDB::add_block( const block& blk +uint64_t BlockchainDB::add_block( const std::pair& blck , size_t block_weight , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector& txs + , const std::vector>& txs ) { + const block &blk = blck.first; + // sanity if (blk.tx_hashes.size() != txs.size()) throw std::runtime_error("Inconsistent tx/hashes sizes"); @@ -221,16 +225,16 @@ uint64_t BlockchainDB::add_block( const block& blk time1 = epee::misc_utils::get_tick_count(); uint64_t num_rct_outs = 0; - add_transaction(blk_hash, blk.miner_tx); + add_transaction(blk_hash, std::make_pair(blk.miner_tx, tx_to_blob(blk.miner_tx))); if (blk.miner_tx.version == 2) num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; - for (const transaction& tx : txs) + for (const std::pair& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); - for (const auto &vout: tx.vout) + for (const auto &vout: tx.first.vout) { if (vout.amount == 0) ++num_rct_outs; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index c3f11ba28..4d0b8935f 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -404,7 +404,7 @@ private: * @param tx_prunable_hash the hash of the prunable part of the transaction * @return the transaction ID */ - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; /** * @brief remove data about a transaction @@ -532,7 +532,7 @@ protected: * @param tx_hash_ptr the hash of the transaction, if already calculated * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated */ - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); + void add_transaction(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric @@ -798,12 +798,12 @@ public: * * @return the height of the chain post-addition */ - virtual uint64_t add_block( const block& blk + virtual uint64_t add_block( const std::pair& blk , size_t block_weight , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector& txs + , const std::vector>& txs ); /** diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 376170a6b..e7ed75c83 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -823,7 +823,7 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } -uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -849,6 +849,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); } + const cryptonote::transaction &tx = txp.first; txindex ti; ti.key = tx_hash; ti.data.tx_id = tx_id; @@ -862,7 +863,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); - cryptonote::blobdata blob = tx_to_blob(tx); + const cryptonote::blobdata &blob = txp.second; MDB_val_sized(blobval, blob); unsigned int unprunable_size = tx.unprunable_size; @@ -3568,8 +3569,8 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - const std::vector& txs) +uint64_t BlockchainLMDB::add_block(const std::pair& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, + const std::vector>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -4488,7 +4489,7 @@ void BlockchainLMDB::migrate_0_1() if (!parse_and_validate_block_from_blob(bd, b)) throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - add_transaction(null_hash, b.miner_tx); + add_transaction(null_hash, std::make_pair(b.miner_tx, tx_to_blob(b.miner_tx))); for (unsigned int j = 0; j(v.mv_data), v.mv_size); if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - add_transaction(null_hash, tx, &b.tx_hashes[j]); + add_transaction(null_hash, std::make_pair(std::move(tx), bd), &b.tx_hashes[j]); result = mdb_cursor_del(c_txs, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 5764f9ae4..2f28c48e2 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -292,12 +292,12 @@ public: virtual bool for_all_outputs(std::function f) const; virtual bool for_all_outputs(uint64_t amount, const std::function &f) const; - virtual uint64_t add_block( const block& blk + virtual uint64_t add_block( const std::pair& blk , size_t block_weight , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector& txs + , const std::vector>& txs ); virtual void set_batch_transactions(bool batch_transactions); @@ -353,7 +353,7 @@ private: virtual void remove_block(); - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 35dfbe673..dcaee79b7 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -102,7 +102,7 @@ public: virtual std::vector> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const { return std::vector>(); } virtual bool has_key_image(const crypto::key_image& img) const { return false; } virtual void remove_block() { } - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const cryptonote::transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) {return 0;} + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) {return 0;} virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) {} virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) {return 0;} virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices) {} -- cgit v1.2.3 From 79b4e9f3779796d9233e3e169f5c1086a49717dd Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 20 Nov 2018 20:19:39 +0000 Subject: save some database calls when getting top block hash and height --- src/blockchain_db/blockchain_db.h | 4 +++- src/blockchain_db/lmdb/db_lmdb.cpp | 4 +++- src/blockchain_db/lmdb/db_lmdb.h | 2 +- src/blockchain_db/testdb.h | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) (limited to 'src/blockchain_db') diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 4d0b8935f..ed13de5b5 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1052,9 +1052,11 @@ public: * * The subclass should return the hash of the most recent block * + * @param block_height if non NULL, returns the height of that block (ie, the blockchain height minus 1) + * * @return the top block's hash */ - virtual crypto::hash top_block_hash() const = 0; + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const = 0; /** * @brief fetch the top block diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index e7ed75c83..c6971c613 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2582,11 +2582,13 @@ std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, c return v; } -crypto::hash BlockchainLMDB::top_block_hash() const +crypto::hash BlockchainLMDB::top_block_hash(uint64_t *block_height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); uint64_t m_height = height(); + if (block_height) + *block_height = m_height - 1; if (m_height != 0) { return get_block_hash_from_height(m_height - 1); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 2f28c48e2..82016c17a 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -233,7 +233,7 @@ public: virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; - virtual crypto::hash top_block_hash() const; + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const; virtual block get_top_block() const; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index dcaee79b7..ac1849b5f 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -79,7 +79,7 @@ public: virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); } virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector(); } virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector(); } - virtual crypto::hash top_block_hash() const { return crypto::hash(); } + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const { if (block_height) *block_height = 0; return crypto::hash(); } virtual cryptonote::block get_top_block() const { return cryptonote::block(); } virtual uint64_t height() const { return 1; } virtual bool tx_exists(const crypto::hash& h) const { return false; } -- cgit v1.2.3