diff options
| -rw-r--r-- | src/blockchain_converter/blockchain_converter.cpp | 14 | ||||
| -rw-r--r-- | src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 371 | ||||
| -rw-r--r-- | src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 53 | ||||
| -rw-r--r-- | src/cryptonote_core/blockchain.cpp | 5 | ||||
| -rw-r--r-- | src/cryptonote_core/blockchain.h | 2 | ||||
| -rw-r--r-- | src/cryptonote_core/blockchain_db.cpp | 68 | ||||
| -rw-r--r-- | src/cryptonote_core/blockchain_db.h | 24 |
7 files changed, 442 insertions, 95 deletions
diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 57822700f..0dfc1a2c3 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014, The Monero Project -// +// Copyright (c) 2014-2015, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 4fa16b23e..a09d9331c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -33,6 +33,7 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "crypto/crypto.h" +#include "profile_tools.h" using epee::string_tools::pod_to_hex; @@ -152,12 +153,13 @@ void BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& blk_hash ) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy<crypto::hash> val_h(get_block_hash(blk)); + MDB_val_copy<crypto::hash> val_h(blk_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); @@ -167,8 +169,11 @@ void BlockchainLMDB::add_block( const block& blk MDB_val_copy<crypto::hash> parent_key(blk.prev_id); MDB_val parent_h; if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); - + } uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; if (parent_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); @@ -177,8 +182,9 @@ void BlockchainLMDB::add_block( const block& blk MDB_val_copy<uint64_t> key(m_height); MDB_val_copy<blobdata> blob(block_to_blob(blk)); - if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0)) - throw0(DB_ERROR("Failed to add block blob to db transaction")); + auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str())); MDB_val_copy<size_t> sz(block_size); if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) @@ -239,12 +245,12 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } -void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) +void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy<crypto::hash> val_h(get_transaction_hash(tx)); + MDB_val_copy<crypto::hash> val_h(tx_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); @@ -382,7 +388,6 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo check_open(); MDB_val_copy<uint64_t> k(out_index); - MDB_val v; /****** Uncomment if ever outputs actually need to be stored in this manner blobdata b; @@ -505,8 +510,8 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) char anything = '\0'; unused.mv_size = sizeof(char); unused.mv_data = &anything; - if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR("Error adding spent key image to db transaction")); + if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -596,15 +601,23 @@ void BlockchainLMDB::check_open() const BlockchainLMDB::~BlockchainLMDB() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + // batch transaction shouldn't be active at this point. If it is, consider it aborted. + if (m_batch_active) + batch_abort(); } -BlockchainLMDB::BlockchainLMDB() +BlockchainLMDB::BlockchainLMDB(bool batch_transactions) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); // initialize folder to something "safe" just in case // someone accidentally misuses this class... m_folder = "thishsouldnotexistbecauseitisgibberish"; m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_batch_active = false; m_height = 0; } @@ -627,6 +640,16 @@ void BlockchainLMDB::open(const std::string& filename) throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); } + // check for existing LMDB files in base directory + boost::filesystem::path old_files = direc.parent_path(); + if (boost::filesystem::exists(old_files / "data.mdb") || + boost::filesystem::exists(old_files / "lock.mdb")) + { + LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str()); + LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); + throw DB_ERROR("Database could not be opened"); + } + m_folder = filename; // set up lmdb environment @@ -672,7 +695,7 @@ void BlockchainLMDB::open(const std::string& filename) lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); *************************************************/ - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); @@ -706,6 +729,13 @@ void BlockchainLMDB::create(const std::string& filename) void BlockchainLMDB::close() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (m_batch_active) + { + LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); + batch_abort(); + } + this->sync(); + // FIXME: not yet thread safe!!! Use with care. mdb_env_close(m_env); } @@ -713,7 +743,14 @@ void BlockchainLMDB::close() void BlockchainLMDB::sync() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - // LMDB documentation leads me to believe this is unnecessary + check_open(); + + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part + // MDB_NOMETASYNC. Force flush to be synchronous. + if (auto result = mdb_env_sync(m_env, true)) + { + throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str())); + } } void BlockchainLMDB::reset() @@ -753,7 +790,6 @@ void BlockchainLMDB::unlock() check_open(); } - bool BlockchainLMDB::block_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -769,7 +805,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const if (get_result == MDB_NOTFOUND) { txn.commit(); - LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) @@ -854,12 +890,17 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_timestamps, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); @@ -867,7 +908,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const uint64_t*)result.mv_data; } @@ -891,12 +933,18 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_sizes, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -904,22 +952,28 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const size_t*)result.mv_data; } difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_diffs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); @@ -927,7 +981,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(difficulty_type*)result.mv_data; } @@ -954,12 +1009,18 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_coins, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -967,7 +1028,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const uint64_t*)result.mv_data; } @@ -977,20 +1039,28 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); } else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block hash from the db")); + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(crypto::hash*)result.mv_data; } @@ -1056,23 +1126,33 @@ uint64_t BlockchainLMDB::height() const return m_height; } - bool BlockchainLMDB::tx_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<crypto::hash> key(h); MDB_val result; - auto get_result = mdb_get(txn, m_txs, &key, &result); + + TIME_MEASURE_START(time1); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; if (get_result == MDB_NOTFOUND) { - txn.commit(); - LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + if (! m_batch_active) + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) @@ -1094,7 +1174,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const MDB_val result; auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); @@ -1107,14 +1187,20 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<crypto::hash> key(h); MDB_val result; - auto get_result = mdb_get(txn, m_txs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx from hash")); @@ -1124,6 +1210,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const transaction tx; if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (! m_batch_active) + txn.commit(); return tx; } @@ -1165,20 +1253,37 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + // If m_batch_active is set, a batch transaction exists beyond this class, + // such as a batch import with verification enabled, or possibly (later) a + // batch network sync. + // + // A regular network sync without batching would be expected to open a new + // read transaction here, as validation is done prior to the write for block + // and tx data. + txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<crypto::hash> key(h); MDB_val result; - auto get_result = mdb_get(txn, m_tx_heights, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); if (get_result == MDB_NOTFOUND) { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; } @@ -1321,13 +1426,18 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy<uint64_t> k(index); MDB_val v; - auto get_result = mdb_get(txn, m_output_txs, &k, &v); + auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -1335,11 +1445,13 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& crypto::hash tx_hash = *(crypto::hash*)v.mv_data; - get_result = mdb_get(txn, m_output_indices, &k, &v); + get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx index")); + if (! m_batch_active) + txn.commit(); return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); } @@ -1350,10 +1462,15 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_output_amounts); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + lmdb_cur cur(*txn_ptr, m_output_amounts); MDB_val_copy<uint64_t> k(amount); MDB_val v; @@ -1382,7 +1499,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con cur.close(); - txn.commit(); + if (! m_batch_active) + txn.commit(); return get_output_tx_and_index_from_global(glob_index); } @@ -1525,6 +1643,91 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const return false; } +void BlockchainLMDB::batch_start() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (m_batch_active) + throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_txn) + throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); + check_open(); + // NOTE: need to make sure it's destroyed properly when done + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + // indicates this transaction is for batch transactions, but not whether it's + // active + m_write_batch_txn.m_batch_txn = true; + m_write_txn = &m_write_batch_txn; + m_batch_active = true; + LOG_PRINT_L3("batch transaction: begin"); +} + +void BlockchainLMDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + LOG_PRINT_L3("batch transaction: committed"); + + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + if (! m_write_batch_txn.m_batch_txn) + throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); + m_write_txn = &m_write_batch_txn; +} + +void BlockchainLMDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + // for destruction of batch transaction + m_write_txn = nullptr; + m_batch_active = false; + LOG_PRINT_L3("batch transaction: end"); +} + +void BlockchainLMDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + // for destruction of batch transaction + m_write_txn = nullptr; + // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. + m_write_batch_txn.abort(); + m_batch_active = false; + LOG_PRINT_L3("batch transaction: aborted"); +} + +void BlockchainLMDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + uint64_t BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty @@ -1534,23 +1737,34 @@ uint64_t BlockchainLMDB::add_block( const block& blk { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } uint64_t num_outputs = m_num_outputs; try { BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); - m_write_txn = NULL; + if (! m_batch_active) + { + m_write_txn = NULL; - txn.commit(); + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } } catch (...) { m_num_outputs = num_outputs; - m_write_txn = NULL; + if (! m_batch_active) + m_write_txn = NULL; throw; } @@ -1559,18 +1773,27 @@ uint64_t BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) { + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } uint64_t num_outputs = m_num_outputs; try { BlockchainDB::pop_block(blk, txs); - m_write_txn = NULL; + if (! m_batch_active) + { + m_write_txn = NULL; - txn.commit(); + txn.commit(); + } } catch (...) { diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 2513aa48a..0b4b5ec1a 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -38,8 +38,23 @@ struct txn_safe txn_safe() : m_txn(NULL) { } ~txn_safe() { - if(m_txn != NULL) + LOG_PRINT_L3("txn_safe: destructor"); + if (m_txn != NULL) { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } mdb_txn_abort(m_txn); } } @@ -60,6 +75,24 @@ struct txn_safe m_txn = NULL; } + // This should only be needed for batch transaction which must be ensured to + // be aborted before mdb_env_close, not after. So we can't rely on + // BlockchainLMDB destructor to call txn_safe destructor, as that's too late + // to properly abort, since mdb_env_close would have been called earlier. + void abort() + { + LOG_PRINT_L3("txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL"); + } + } + operator MDB_txn*() { return m_txn; @@ -71,13 +104,14 @@ struct txn_safe } MDB_txn* m_txn; + bool m_batch_txn = false; }; class BlockchainLMDB : public BlockchainDB { public: - BlockchainLMDB(); + BlockchainLMDB(bool batch_transactions=false); ~BlockchainLMDB(); virtual void open(const std::string& filename); @@ -177,6 +211,12 @@ public: , const std::vector<transaction>& txs ); + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + virtual void pop_block(block& blk, std::vector<transaction>& txs); private: @@ -184,11 +224,12 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& block_hash ); virtual void remove_block(); - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx); + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); @@ -263,7 +304,11 @@ private: uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe* m_write_txn; + txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress }; } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 525c7f36e..b2d979432 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -237,8 +237,9 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) m_testnet = testnet; boost::filesystem::path folder(m_config_folder); + folder /= "lmdb"; - LOG_PRINT_L0("Loading blockchain..."); + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); //FIXME: update filename for BlockchainDB const std::string filename = folder.string(); @@ -2288,7 +2289,7 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -void Blockchain::check_against_checkpoints(checkpoints& points, bool enforce) +void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce) { const auto& pts = points.get_points(); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 6ab963ac7..3e8a2de31 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -141,7 +141,7 @@ namespace cryptonote void print_blockchain_index(); void print_blockchain_outs(const std::string& file); - void check_against_checkpoints(checkpoints& points, bool enforce); + void check_against_checkpoints(const checkpoints& points, bool enforce); void set_enforce_dns_checkpoints(bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 0ee10bd4d..fc8aeef4e 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -28,6 +28,7 @@ #include "cryptonote_core/blockchain_db.h" #include "cryptonote_format_utils.h" +#include "profile_tools.h" using epee::string_tools::pod_to_hex; @@ -41,11 +42,21 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) { - crypto::hash tx_hash = get_transaction_hash(tx); + crypto::hash tx_hash; + if (!tx_hash_ptr) + { + // should only need to compute hash for miner transactions + tx_hash = get_transaction_hash(tx); + LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash); + } + else + { + tx_hash = *tx_hash_ptr; + } - add_transaction_data(blk_hash, tx); + add_transaction_data(blk_hash, tx, tx_hash); // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index @@ -73,18 +84,33 @@ uint64_t BlockchainDB::add_block( const block& blk , const std::vector<transaction>& txs ) { + TIME_MEASURE_START(time1); crypto::hash blk_hash = get_block_hash(blk); + TIME_MEASURE_FINISH(time1); + time_blk_hash += time1; // call out to subclass implementation to add the block & metadata - add_block(blk, block_size, cumulative_difficulty, coins_generated); + time1 = epee::misc_utils::get_tick_count(); + add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); + TIME_MEASURE_FINISH(time1); + time_add_block1 += time1; // call out to add the transactions + time1 = epee::misc_utils::get_tick_count(); add_transaction(blk_hash, blk.miner_tx); + int tx_i = 0; + crypto::hash tx_hash = null_hash; for (const transaction& tx : txs) { - add_transaction(blk_hash, tx); + tx_hash = blk.tx_hashes[tx_i]; + add_transaction(blk_hash, tx, &tx_hash); + ++tx_i; } + TIME_MEASURE_FINISH(time1); + time_add_transaction += time1; + + ++num_calls; return height(); } @@ -119,4 +145,36 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) remove_transaction_data(tx_hash, tx); } +void BlockchainDB::reset_stats() +{ + num_calls = 0; + time_blk_hash = 0; + time_tx_exists = 0; + time_add_block1 = 0; + time_add_transaction = 0; + time_commit1 = 0; +} + +void BlockchainDB::show_stats() +{ + LOG_PRINT_L1(ENDL + << "*********************************" + << ENDL + << "num_calls: " << num_calls + << ENDL + << "time_blk_hash: " << time_blk_hash << "ms" + << ENDL + << "time_tx_exists: " << time_tx_exists << "ms" + << ENDL + << "time_add_block1: " << time_add_block1 << "ms" + << ENDL + << "time_add_transaction: " << time_add_transaction << "ms" + << ENDL + << "time_commit1: " << time_commit1 << "ms" + << ENDL + << "*********************************" + << ENDL + ); +} + } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index db56c7c07..2a7fa8f82 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -265,13 +265,14 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& blk_hash ) = 0; // tells the subclass to remove data about the top block virtual void remove_block() = 0; // tells the subclass to store the transaction and its metadata - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0; + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; // tells the subclass to remove data about a transaction virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; @@ -296,17 +297,33 @@ private: void pop_block(); // helper function for add_transactions, to add each individual tx - void add_transaction(const crypto::hash& blk_hash, const transaction& tx); + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); // helper function to remove transaction from blockchain void remove_transaction(const crypto::hash& tx_hash); + uint64_t num_calls = 0; + uint64_t time_blk_hash = 0; + uint64_t time_add_block1 = 0; + uint64_t time_add_transaction = 0; + + +protected: + + mutable uint64_t time_tx_exists = 0; + uint64_t time_commit1 = 0; + public: // virtual dtor virtual ~BlockchainDB() { }; + // reset profiling stats + void reset_stats(); + + // show profiling stats + void show_stats(); // open the db at location <filename>, or create it if there isn't one. virtual void open(const std::string& filename) = 0; @@ -336,6 +353,9 @@ public: // release db lock virtual void unlock() = 0; + virtual void batch_start() = 0; + virtual void batch_stop() = 0; + virtual void set_batch_transactions(bool) = 0; // adds a block with the given metadata to the top of the blockchain, returns the new height // NOTE: subclass implementations of this (or the functions it calls) need |