diff options
Diffstat (limited to 'src/blockchain_db')
| -rw-r--r-- | src/blockchain_db/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | src/blockchain_db/berkeleydb/db_bdb.cpp | 904 | ||||
| -rw-r--r-- | src/blockchain_db/berkeleydb/db_bdb.h | 172 | ||||
| -rw-r--r-- | src/blockchain_db/blockchain_db.cpp | 2 | ||||
| -rw-r--r-- | src/blockchain_db/blockchain_db.h | 27 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.cpp | 466 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.h | 35 |
7 files changed, 1005 insertions, 605 deletions
diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index adbe804aa..7301cab85 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -31,7 +31,7 @@ set(blockchain_db_sources lmdb/db_lmdb.cpp ) -if (NOT STATIC) +if (BERKELEY_DB) set(blockchain_db_sources ${blockchain_db_sources} berkeleydb/db_bdb.cpp @@ -46,7 +46,7 @@ set(blockchain_db_private_headers lmdb/db_lmdb.h ) -if (NOT STATIC) +if (BERKELEY_DB) set(blockchain_db_private_headers ${blockchain_db_private_headers} berkeleydb/db_bdb.h diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index a8163d4b1..ad3febbd1 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -36,6 +36,7 @@ #include "profile_tools.h" using epee::string_tools::pod_to_hex; +#define DB_DEFAULT_TX (m_write_txn != nullptr ? *m_write_txn : (DbTxn*) nullptr) namespace { @@ -117,12 +118,24 @@ const char* const BDB_OUTPUT_KEYS = "output_keys"; const char* const BDB_SPENT_KEYS = "spent_keys"; -const int BUFFER_LENGTH = 4 * 1024 * 1024; +const unsigned int MB = 1024 * 1024; +// ND: FIXME: db keeps running out of locks when doing full syncs. Possible bug??? Set it to 5K for now. +const unsigned int DB_MAX_LOCKS = 5000; +const unsigned int DB_BUFFER_LENGTH = 32 * MB; +// 256MB cache adjust as necessary using DB_CONFIG +const unsigned int DB_DEF_CACHESIZE = 256 * MB; + +#if defined(BDB_BULK_CAN_THREAD) +const unsigned int DB_BUFFER_COUNT = std::thread::hardware_concurrency(); +#else +const unsigned int DB_BUFFER_COUNT = 1; +#endif template<typename T> struct Dbt_copy: public Dbt { - Dbt_copy(const T &t): t_copy(t) + Dbt_copy(const T &t) : + t_copy(t) { init(); } @@ -151,7 +164,8 @@ private: template<> struct Dbt_copy<cryptonote::blobdata>: public Dbt { - Dbt_copy(const cryptonote::blobdata &bd) : m_data(new char[bd.size()]) + Dbt_copy(const cryptonote::blobdata &bd) : + m_data(new char[bd.size()]) { memcpy(m_data.get(), bd.data(), bd.size()); set_data(m_data.get()); @@ -185,25 +199,20 @@ struct Dbt_safe : public Dbt namespace cryptonote { -void BlockchainBDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const crypto::hash& blk_hash - ) +void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); Dbt_copy<crypto::hash> val_h(blk_hash); - if (m_block_heights->exists(*m_write_txn, &val_h, 0) == 0) + if (m_block_heights->exists(DB_DEFAULT_TX, &val_h, 0) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); if (m_height > 0) { Dbt_copy<crypto::hash> parent_key(blk.prev_id); Dbt_copy<uint32_t> parent_h; - if (m_block_heights->get(*m_write_txn, &parent_key, &parent_h, 0)) + if (m_block_heights->get(DB_DEFAULT_TX, &parent_key, &parent_h, 0)) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); @@ -217,30 +226,30 @@ void BlockchainBDB::add_block( const block& blk Dbt_copy<uint32_t> key(m_height + 1); Dbt_copy<blobdata> blob(block_to_blob(blk)); - auto res = m_blocks->put(*m_write_txn, &key, &blob, 0); + auto res = m_blocks->put(DB_DEFAULT_TX, &key, &blob, 0); if (res) throw0(DB_ERROR("Failed to add block blob to db transaction.")); Dbt_copy<size_t> sz(block_size); - if (m_block_sizes->put(*m_write_txn, &key, &sz, 0)) + if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0)) throw0(DB_ERROR("Failed to add block size to db transaction.")); Dbt_copy<uint64_t> ts(blk.timestamp); - if (m_block_timestamps->put(*m_write_txn, &key, &ts, 0)) + if (m_block_timestamps->put(DB_DEFAULT_TX, &key, &ts, 0)) throw0(DB_ERROR("Failed to add block timestamp to db transaction.")); Dbt_copy<difficulty_type> diff(cumulative_difficulty); - if (m_block_diffs->put(*m_write_txn, &key, &diff, 0)) + if (m_block_diffs->put(DB_DEFAULT_TX, &key, &diff, 0)) throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction.")); Dbt_copy<uint64_t> coinsgen(coins_generated); - if (m_block_coins->put(*m_write_txn, &key, &coinsgen, 0)) + if (m_block_coins->put(DB_DEFAULT_TX, &key, &coinsgen, 0)) throw0(DB_ERROR("Failed to add block total generated coins to db transaction.")); - if (m_block_heights->put(*m_write_txn, &val_h, &key, 0)) + if (m_block_heights->put(DB_DEFAULT_TX, &val_h, &key, 0)) throw0(DB_ERROR("Failed to add block height by hash to db transaction.")); - if (m_block_hashes->put(*m_write_txn, &key, &val_h, 0)) + if (m_block_hashes->put(DB_DEFAULT_TX, &key, &val_h, 0)) throw0(DB_ERROR("Failed to add block hash to db transaction.")); } @@ -254,28 +263,28 @@ void BlockchainBDB::remove_block() Dbt_copy<uint32_t> k(m_height); Dbt_copy<crypto::hash> h; - if (m_block_hashes->get(*m_write_txn, &k, &h, 0)) + if (m_block_hashes->get(DB_DEFAULT_TX, &k, &h, 0)) throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); - if (m_blocks->del(*m_write_txn, &k, 0)) + if (m_blocks->del(DB_DEFAULT_TX, &k, 0)) throw1(DB_ERROR("Failed to add removal of block to db transaction")); - if (m_block_sizes->del(*m_write_txn, &k, 0)) + if (m_block_sizes->del(DB_DEFAULT_TX, &k, 0)) throw1(DB_ERROR("Failed to add removal of block size to db transaction")); - if (m_block_diffs->del(*m_write_txn, &k, 0)) + if (m_block_diffs->del(DB_DEFAULT_TX, &k, 0)) throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); - if (m_block_coins->del(*m_write_txn, &k, 0)) + if (m_block_coins->del(DB_DEFAULT_TX, &k, 0)) throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); - if (m_block_timestamps->del(*m_write_txn, &k, 0)) + if (m_block_timestamps->del(DB_DEFAULT_TX, &k, 0)) throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); - if (m_block_heights->del(*m_write_txn, &h, 0)) + if (m_block_heights->del(DB_DEFAULT_TX, &h, 0)) throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); - if (m_block_hashes->del(*m_write_txn, &k, 0)) + if (m_block_hashes->del(DB_DEFAULT_TX, &k, 0)) throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } @@ -286,19 +295,19 @@ void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const tra Dbt_copy<crypto::hash> val_h(tx_hash); - if (m_txs->exists(*m_write_txn, &val_h, 0) == 0) + if (m_txs->exists(DB_DEFAULT_TX, &val_h, 0) == 0) throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); Dbt_copy<blobdata> blob(tx_to_blob(tx)); - if (m_txs->put(*m_write_txn, &val_h, &blob, 0)) + if (m_txs->put(DB_DEFAULT_TX, &val_h, &blob, 0)) throw0(DB_ERROR("Failed to add tx blob to db transaction")); Dbt_copy<uint64_t> height(m_height + 1); - if (m_tx_heights->put(*m_write_txn, &val_h, &height, 0)) + if (m_tx_heights->put(DB_DEFAULT_TX, &val_h, &height, 0)) throw0(DB_ERROR("Failed to add tx block height to db transaction")); Dbt_copy<uint64_t> unlock_time(tx.unlock_time); - if (m_tx_unlocks->put(*m_write_txn, &val_h, &unlock_time, 0)) + if (m_tx_unlocks->put(DB_DEFAULT_TX, &val_h, &unlock_time, 0)) throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); } @@ -308,24 +317,24 @@ void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const t check_open(); Dbt_copy<crypto::hash> val_h(tx_hash); - if (m_txs->exists(*m_write_txn, &val_h, 0)) + if (m_txs->exists(DB_DEFAULT_TX, &val_h, 0)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - if (m_txs->del(*m_write_txn, &val_h, 0)) + if (m_txs->del(DB_DEFAULT_TX, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (m_tx_unlocks->del(*m_write_txn, &val_h, 0)) + if (m_tx_unlocks->del(DB_DEFAULT_TX, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (m_tx_heights->del(*m_write_txn, &val_h, 0)) + if (m_tx_heights->del(DB_DEFAULT_TX, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); remove_tx_outputs(tx_hash, tx); - if (m_tx_outputs->del(*m_write_txn, &val_h, 0)) + if (m_tx_outputs->del(DB_DEFAULT_TX, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); } -void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -333,23 +342,28 @@ void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_out Dbt_copy<uint32_t> k(m_num_outputs + 1); Dbt_copy<crypto::hash> v(tx_hash); - if (m_output_txs->put(*m_write_txn, &k, &v, 0)) + if (m_output_txs->put(DB_DEFAULT_TX, &k, &v, 0)) throw0(DB_ERROR("Failed to add output tx hash to db transaction")); - if (m_tx_outputs->put(*m_write_txn, &v, &k, 0)) + if (m_tx_outputs->put(DB_DEFAULT_TX, &v, &k, 0)) throw0(DB_ERROR("Failed to add tx output index to db transaction")); Dbt_copy<uint64_t> val_local_index(local_index); - if (m_output_indices->put(*m_write_txn, &k, &val_local_index, 0)) + if (m_output_indices->put(DB_DEFAULT_TX, &k, &val_local_index, 0)) throw0(DB_ERROR("Failed to add tx output index to db transaction")); Dbt_copy<uint64_t> val_amount(tx_output.amount); - if (m_output_amounts->put(*m_write_txn, &val_amount, &k, 0)) + if (m_output_amounts->put(DB_DEFAULT_TX, &val_amount, &k, 0)) throw0(DB_ERROR("Failed to add output amount to db transaction.")); if (tx_output.target.type() == typeid(txout_to_key)) { - Dbt_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); - if (m_output_keys->put(*m_write_txn, &k, &val_pubkey, 0)) + output_data_t od; + od.pubkey = boost::get < txout_to_key > (tx_output.target).key; + od.unlock_time = unlock_time; + od.height = m_height; + + Dbt_copy<output_data_t> data(od); + if (m_output_keys->put(DB_DEFAULT_TX, &k, &data, 0)) throw0(DB_ERROR("Failed to add output pubkey to db transaction")); } @@ -360,7 +374,7 @@ void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transac { LOG_PRINT_L3("BlockchainBDB::" << __func__); - bdb_cur cur(*m_write_txn, m_tx_outputs); + bdb_cur cur(DB_DEFAULT_TX, m_tx_outputs); Dbt_copy<crypto::hash> k(tx_hash); Dbt_copy<uint32_t> v; @@ -407,7 +421,7 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou Dbt_copy<uint32_t> k(out_index); - auto result = m_output_indices->del(*m_write_txn, &k, 0); + auto result = m_output_indices->del(DB_DEFAULT_TX, &k, 0); if (result == DB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); @@ -417,7 +431,7 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); } - result = m_output_txs->del(*m_write_txn, &k, 0); + result = m_output_txs->del(DB_DEFAULT_TX, &k, 0); // if (result != 0 && result != DB_NOTFOUND) // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); if (result == DB_NOTFOUND) @@ -429,7 +443,7 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); } - result = m_output_keys->del(*m_write_txn, &k, 0); + result = m_output_keys->del(DB_DEFAULT_TX, &k, 0); if (result == DB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); @@ -447,7 +461,7 @@ void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_cur cur(*m_write_txn, m_output_amounts); + bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); Dbt_copy<uint64_t> k(amount); Dbt_copy<uint32_t> v; @@ -497,11 +511,11 @@ void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) check_open(); Dbt_copy<crypto::key_image> val_key(k_image); - if (m_spent_keys->exists(*m_write_txn, &val_key, 0) == 0) + if (m_spent_keys->exists(DB_DEFAULT_TX, &val_key, 0) == 0) throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); Dbt_copy<char> val('\0'); - if (m_spent_keys->put(*m_write_txn, &val_key, &val, 0)) + if (m_spent_keys->put(DB_DEFAULT_TX, &val_key, &val, 0)) throw1(DB_ERROR("Error adding spent key image to db transaction.")); } @@ -511,7 +525,7 @@ void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) check_open(); Dbt_copy<crypto::key_image> k(k_image); - auto result = m_spent_keys->del(*m_write_txn, &k, 0); + auto result = m_spent_keys->del(DB_DEFAULT_TX, &k, 0); if (result != 0 && result != DB_NOTFOUND) throw1(DB_ERROR("Error adding removal of key image to db transaction")); } @@ -539,44 +553,19 @@ tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const return o; } -uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - bdb_cur cur(txn, m_output_amounts); - - Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint32_t> v; - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) + std::vector < uint64_t > offsets; + std::vector < uint64_t > global_indices; + offsets.push_back(index); + get_output_global_indices(amount, offsets, global_indices); + if (!global_indices.size()) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - db_recno_t num_elems; - cur->count(&num_elems, 0); - - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - for (uint64_t i = 0; i < index; ++i) - { - cur->get(&k, &v, DB_NEXT_DUP); - } - - uint64_t glob_index = v; - - cur.close(); - - txn.commit(); - - return glob_index; + return global_indices[0]; } void BlockchainBDB::check_open() const @@ -589,24 +578,22 @@ void BlockchainBDB::check_open() const BlockchainBDB::~BlockchainBDB() { LOG_PRINT_L3("BlockchainBDB::" << __func__); - if(m_buffer != NULL) - free(m_buffer); - m_buffer = NULL; + if (m_open) { close(); } } -BlockchainBDB::BlockchainBDB(bool batch_transactions) +BlockchainBDB::BlockchainBDB(bool batch_transactions) : + m_buffer(DB_BUFFER_COUNT, DB_BUFFER_LENGTH) { - m_buffer = malloc(BUFFER_LENGTH); LOG_PRINT_L3("BlockchainBDB::" << __func__); // initialize folder to something "safe" just in case // someone accidentally misuses this class... m_folder = "thishsouldnotexistbecauseitisgibberish"; m_open = false; - + m_run_checkpoint = 0; m_batch_transactions = batch_transactions; m_write_txn = nullptr; m_height = 0; @@ -639,12 +626,21 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) //Create BerkeleyDB environment m_env = new DbEnv(0); // no flags needed for DbEnv - uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK - | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER - | DB_THREAD; + uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER | DB_THREAD; + // Set some default values for these parameters. + // They can be overridden using the DB_CONFIG file. + m_env->set_cachesize(0, DB_DEF_CACHESIZE, 1); + m_env->set_lk_max_locks(DB_MAX_LOCKS); + m_env->set_lk_max_lockers(DB_MAX_LOCKS); + m_env->set_lk_max_objects(DB_MAX_LOCKS); + // last parameter left 0, files will be created with default rw access m_env->open(filename.c_str(), db_env_open_flags, 0); + m_env->set_flags(db_flags, 1); + + if(m_auto_remove_logs) + m_env->log_set_config(DB_LOG_AUTO_REMOVE, 1); // begin transaction to init dbs bdb_txn_safe txn; @@ -686,7 +682,7 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) m_output_txs->set_re_len(sizeof(crypto::hash)); m_output_indices->set_re_len(sizeof(uint64_t)); - m_output_keys->set_re_len(sizeof(crypto::public_key)); + m_output_keys->set_re_len(sizeof(output_data_t)); //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) // for the RECNO databases. We shouldn't as we're only @@ -717,21 +713,29 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_HASH, DB_CREATE, 0); + txn.commit(); + DB_BTREE_STAT* stats; // DB_FAST_STAT can apparently cause an incorrect number of records // to be returned. The flag should be set to 0 instead if this proves // to be the case. - m_blocks->stat(txn, &stats, DB_FAST_STAT); + + // ND: The bug above can occur when a block is popped and the application + // exits without pushing a new block to the db. Set txn to NULL and DB_FAST_STAT + // to zero (0) for reliability. + m_blocks->stat(NULL, &stats, 0); m_height = stats->bt_nkeys; delete stats; // see above comment about DB_FAST_STAT - m_output_indices->stat(txn, &stats, DB_FAST_STAT); + m_output_indices->stat(NULL, &stats, 0); m_num_outputs = stats->bt_nkeys; delete stats; - txn.commit(); + // run checkpoint thread + m_run_checkpoint = true; + m_checkpoint_thread.reset(new boost::thread(&BlockchainBDB::checkpoint_worker, this)); } catch (const std::exception& e) { @@ -746,6 +750,10 @@ void BlockchainBDB::close() LOG_PRINT_L3("BlockchainBDB::" << __func__); this->sync(); + m_run_checkpoint = false; + m_checkpoint_thread->join(); + m_checkpoint_thread.reset(); + // FIXME: not yet thread safe!!! Use with care. m_open = false; m_env->close(DB_FORCESYNC); @@ -786,7 +794,7 @@ void BlockchainBDB::sync() void BlockchainBDB::reset() { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainBDB::" << __func__); // TODO: this } @@ -853,7 +861,7 @@ std::vector<std::string> BlockchainBDB::get_filenames() const std::vector<std::string> full_paths; -for (auto& filename : filenames) + for (auto& filename : filenames) { boost::filesystem::path p(m_folder); p /= filename; @@ -890,23 +898,17 @@ bool BlockchainBDB::block_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<crypto::hash> key(h); - auto get_result = m_block_heights->exists(txn, &key, 0); + auto get_result = m_block_heights->exists(DB_DEFAULT_TX, &key, 0); if (get_result == DB_NOTFOUND) { - txn.commit(); LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) throw0(DB_ERROR("DB error attempting to fetch block index from hash")); - txn.commit(); return true; } @@ -923,22 +925,16 @@ uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<crypto::hash> key(h); - Dbt_copy<uint64_t> result; + Dbt_copy<uint32_t> result; - auto get_result = m_block_heights->get(txn, &key, &result, 0); + auto get_result = m_block_heights->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); - txn.commit(); - - return (uint64_t)result - 1; + return result - 1; } block_header BlockchainBDB::get_block_header(const crypto::hash& h) const @@ -955,13 +951,9 @@ block BlockchainBDB::get_block_from_height(const uint64_t& height) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<uint32_t> key(height + 1); Dbt_safe result; - auto get_result = m_blocks->get(txn, &key, &result, 0); + auto get_result = m_blocks->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str())); @@ -969,8 +961,6 @@ block BlockchainBDB::get_block_from_height(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block from the db")); - txn.commit(); - blobdata bd; bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size()); @@ -986,13 +976,9 @@ uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<uint32_t> key(height + 1); Dbt_copy<uint64_t> result; - auto get_result = m_block_timestamps->get(txn, &key, &result, 0); + auto get_result = m_block_timestamps->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); @@ -1000,7 +986,6 @@ uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - txn.commit(); return result; } @@ -1023,13 +1008,9 @@ size_t BlockchainBDB::get_block_size(const uint64_t& height) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<uint32_t> key(height + 1); Dbt_copy<size_t> result; - auto get_result = m_block_sizes->get(txn, &key, &result, 0); + auto get_result = m_block_sizes->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -1037,7 +1018,6 @@ size_t BlockchainBDB::get_block_size(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - txn.commit(); return result; } @@ -1046,13 +1026,9 @@ difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& h LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<uint32_t> key(height + 1); Dbt_copy<difficulty_type> result; - auto get_result = m_block_diffs->get(txn, &key, &result, 0); + auto get_result = m_block_diffs->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); @@ -1060,7 +1036,6 @@ difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& h else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - txn.commit(); return result; } @@ -1086,13 +1061,9 @@ uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<uint32_t> key(height + 1); Dbt_copy<uint64_t> result; - auto get_result = m_block_coins->get(txn, &key, &result, 0); + auto get_result = m_block_coins->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -1100,7 +1071,6 @@ uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - txn.commit(); return result; } @@ -1109,13 +1079,9 @@ crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) c LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<uint32_t> key(height + 1); Dbt_copy<crypto::hash> result; - auto get_result = m_block_hashes->get(txn, &key, &result, 0); + auto get_result = m_block_hashes->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); @@ -1123,7 +1089,6 @@ crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) c else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block hash from the db.")); - txn.commit(); return result; } @@ -1194,19 +1159,14 @@ bool BlockchainBDB::tx_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<crypto::hash> key(h); TIME_MEASURE_START(time1); - auto get_result = m_txs->exists(txn, &key, 0); + auto get_result = m_txs->exists(DB_DEFAULT_TX, &key, 0); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; if (get_result == DB_NOTFOUND) { - txn.commit(); LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } @@ -1218,16 +1178,12 @@ bool BlockchainBDB::tx_exists(const crypto::hash& h) const uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<crypto::hash> key(h); Dbt_copy<uint64_t> result; - auto get_result = m_tx_unlocks->get(txn, &key, &result, 0); + auto get_result = m_tx_unlocks->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1241,13 +1197,9 @@ transaction BlockchainBDB::get_tx(const crypto::hash& h) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<crypto::hash> key(h); Dbt_safe result; - auto get_result = m_txs->get(txn, &key, &result, 0); + auto get_result = m_txs->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1260,8 +1212,6 @@ transaction BlockchainBDB::get_tx(const crypto::hash& h) const if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - txn.commit(); - return tx; } @@ -1270,21 +1220,15 @@ uint64_t BlockchainBDB::get_tx_count() const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - DB_BTREE_STAT* stats; // DB_FAST_STAT can apparently cause an incorrect number of records // to be returned. The flag should be set to 0 instead if this proves // to be the case. - m_txs->stat(txn, &stats, DB_FAST_STAT); + m_txs->stat(DB_DEFAULT_TX, &stats, DB_FAST_STAT); auto num_txs = stats->bt_nkeys; delete stats; - txn.commit(); - return num_txs; } @@ -1307,13 +1251,9 @@ uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<crypto::hash> key(h); Dbt_copy<uint64_t> result; - auto get_result = m_tx_heights->get(txn, &key, &result, 0); + auto get_result = m_tx_heights->get(DB_DEFAULT_TX, &key, &result, 0); if (get_result == DB_NOTFOUND) { throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); @@ -1321,8 +1261,6 @@ uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); - txn.commit(); - return (uint64_t)result - 1; } @@ -1344,14 +1282,10 @@ uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - bdb_cur cur(txn, m_output_amounts); + bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint64_t> v; + Dbt_copy<uint32_t> v; auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) { @@ -1363,25 +1297,19 @@ uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const db_recno_t num_elems = 0; cur->count(&num_elems, 0); - txn.commit(); + cur.close(); return num_elems; } -crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +output_data_t BlockchainBDB::get_output_key(const uint64_t& global_index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - uint64_t glob_index = get_output_global_index(amount, index); - - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - Dbt_copy<uint32_t> k(glob_index); - Dbt_copy<crypto::public_key> v; - auto get_result = m_output_keys->get(txn, &k, &v, 0); + Dbt_copy<uint32_t> k(global_index); + Dbt_copy<output_data_t> v; + auto get_result = m_output_keys->get(DB_DEFAULT_TX, &k, &v, 0); if (get_result == DB_NOTFOUND) throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) @@ -1390,6 +1318,15 @@ crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const u return v; } +output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + return get_output_key(glob_index); +} + // As this is not used, its return is now a blank output. // This will save on space in the db. tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const @@ -1406,230 +1343,17 @@ tx_out BlockchainBDB::get_output(const uint64_t& index) const return tx_out(); } -void BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, - std::vector<uint64_t> &offsets, - std::vector<tx_out_index> &indices) const +tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - bdb_cur cur(txn, m_output_amounts); - uint64_t max = 0; - for(const uint64_t& index : offsets) - { - if(index > max) - max = index; - } - - // ??? might be a bug, don't always treat as uint64_t - #define DBT_VALUE(dbt) v.get_size() == sizeof(uint64_t) ? \ - *((uint64_t *)v.get_data()) : *((uint32_t *)v.get_data()) \ - - // get returned keypairs count - #define DB_COUNT_RECORDS(dbt, cnt) \ - do { \ - uint32_t *_p = (uint32_t *) ((uint8_t *)(dbt)->data + \ - (dbt)->ulen - sizeof(uint32_t)); \ - cnt = 0; \ - while(*_p != (uint32_t) -1) { \ - _p -= 2; \ - ++cnt; \ - } \ - } while(0); \ - - Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint64_t> v; - uint64_t buflen = 0; - uint64_t t_dbmul = 0; - uint64_t t_dbscan = 0; - TIME_MEASURE_START(db2); - if(max <= 1) - { - for (const uint64_t& index : offsets) - { - TIME_MEASURE_START(t_seek); - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - for (uint64_t i = 0; i < index; ++i) - { - cur->get(&k, &v, DB_NEXT_DUP); - } - - uint64_t glob_index = DBT_VALUE(v); - - LOG_PRINT_L1("L0->v: " << glob_index); - tx_out_index res = get_output_tx_and_index_from_global(glob_index); - indices.push_back(res); - - TIME_MEASURE_FINISH(t_seek); - } - } - else - { - // setup a 256KB minimum buffer size - uint32_t pagesize = 256 * 1024; - - // Retrieve only a suitable portion of the kvp data, up to somewhere near - // the maximum offset value being retrieved - buflen = (max + 1) * 4 * sizeof(uint64_t); - buflen = ((buflen / pagesize) + ((buflen % pagesize) > 0 ? 1 : 0)) * pagesize; - bool singlebuff = buflen <= BUFFER_LENGTH; - buflen = buflen < BUFFER_LENGTH ? buflen : BUFFER_LENGTH; - - Dbt data; - data.set_data(m_buffer); - data.set_ulen(buflen); - data.set_size(buflen); - data.set_flags(DB_DBT_USERMEM); - - uint32_t curcount = 0; - uint32_t blockstart = 0; - for (const uint64_t& index : offsets) - { - // fixme! for whatever reason, the first call to DB_MULTIPLE | DB_SET does not - // retrieve the first value. - if(index <= 1) - { - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - if(index > 0) - cur->get(&k, &v, DB_NEXT_DUP); - } - else - { - while(index >= curcount) - { - TIME_MEASURE_START(t_db1); - try - { - cur->get(&k, &data, DB_MULTIPLE | (curcount == 0 ? DB_SET : DB_NEXT_DUP)); - blockstart = curcount; - - int count = 0; - // fixme! this might be slow on some systems. - DB_COUNT_RECORDS((DBT *) &data, count); - curcount += count; - } - catch (const std::exception &e) - { - LOG_PRINT_L0("DB_EXCEPTION: " << e.what()); - } - - TIME_MEASURE_FINISH(t_db1); - t_dbmul += t_db1; - if(singlebuff) - break; - } - - LOG_PRINT_L1("Records returned: " << curcount << " Index: " << index); - TIME_MEASURE_START(t_db2); - DBT *pdata = (DBT *) &data; - - uint8_t *value; - uint64_t dlen = 0; - - void *pbase = ((uint8_t *)(pdata->data)) + pdata->ulen - sizeof(uint32_t); - uint32_t *p = (uint32_t *) pbase; - if (*p == (uint32_t) -1) - { - value = NULL; - } - else - { - p -= (index - blockstart) * 2; // index * 4 + 2; <- if DB_MULTIPLE_KEY - value = (uint8_t *) pdata->data + *p--; - dlen = *p--; - if (value == (uint8_t *) pdata->data) - value = NULL; - } - - if (value != NULL) - { - v = dlen == sizeof(uint64_t) ? *((uint64_t *) value) - : *((uint32_t *) value); - } - TIME_MEASURE_FINISH(t_db2); - t_dbscan += t_db2; - } - - uint64_t glob_index = DBT_VALUE(v); - - LOG_PRINT_L1("L1->v: " << glob_index); - tx_out_index res = get_output_tx_and_index_from_global(glob_index); - indices.push_back(res); - } - } - - TIME_MEASURE_FINISH(db2); - - LOG_PRINT_L1("blen: " << buflen << " db1: " << t_dbmul << " db2: " << t_dbscan); - - cur.close(); - txn.commit(); -} - -tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - bdb_cur cur(txn, m_output_amounts); - - Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint64_t> v; - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) + std::vector < uint64_t > offsets; + std::vector<tx_out_index> indices; + offsets.push_back(index); + get_output_tx_and_index(amount, offsets, indices); + if (!indices.size()) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - for (uint64_t i = 0; i < index; ++i) - { - cur->get(&k, &v, DB_NEXT_DUP); - } - - uint64_t glob_index = v; - - cur.close(); - - txn.commit(); - return get_output_tx_and_index_from_global(glob_index); + return indices[0]; } std::vector<uint64_t> BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const @@ -1638,14 +1362,10 @@ std::vector<uint64_t> BlockchainBDB::get_tx_output_indices(const crypto::hash& h check_open(); std::vector<uint64_t> index_vec; - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - bdb_cur cur(txn, m_tx_outputs); + bdb_cur cur(DB_DEFAULT_TX, m_tx_outputs); Dbt_copy<crypto::hash> k(h); - Dbt_copy<uint64_t> v; + Dbt_copy<uint32_t> v; auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); @@ -1662,7 +1382,6 @@ std::vector<uint64_t> BlockchainBDB::get_tx_output_indices(const crypto::hash& h } cur.close(); - txn.commit(); return index_vec; } @@ -1680,10 +1399,6 @@ std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto:: transaction tx = get_tx(h); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - uint64_t i = 0; uint64_t global_index; for (const auto& vout : tx.vout) @@ -1692,10 +1407,10 @@ std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto:: global_index = index_vec[i]; - bdb_cur cur(txn, m_output_amounts); + bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint64_t> v; + Dbt_copy<uint32_t> v; auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) @@ -1728,7 +1443,6 @@ std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto:: { // not found cur.close(); - txn.commit(); throw1(OUTPUT_DNE("specified output not found in db")); } @@ -1736,8 +1450,6 @@ std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto:: ++i; } - txn.commit(); - return index_vec2; } @@ -1747,14 +1459,10 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<uint32_t> k(index); Dbt_copy<crypto::hash > v; - auto get_result = m_output_txs->get(txn, &k, &v, 0); + auto get_result = m_output_txs->get(DB_DEFAULT_TX, &k, &v, 0); if (get_result == DB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -1763,14 +1471,12 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& crypto::hash tx_hash = v; Dbt_copy<uint64_t> result; - get_result = m_output_indices->get(txn, &k, &result, 0); + get_result = m_output_indices->get(DB_DEFAULT_TX, &k, &result, 0); if (get_result == DB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx index")); - txn.commit(); - return tx_out_index(tx_hash, result); } @@ -1779,18 +1485,12 @@ bool BlockchainBDB::has_key_image(const crypto::key_image& img) const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy<crypto::key_image> val_key(img); - if (m_spent_keys->exists(txn, &val_key, 0) == 0) + if (m_spent_keys->exists(DB_DEFAULT_TX, &val_key, 0) == 0) { - txn.commit(); return true; } - txn.commit(); return false; } @@ -1819,17 +1519,12 @@ void BlockchainBDB::batch_abort() void BlockchainBDB::set_batch_transactions(bool batch_transactions) { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainBDB::" << __func__); m_batch_transactions = batch_transactions; LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } -uint64_t BlockchainBDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector<transaction>& txs - ) +uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1888,4 +1583,295 @@ void BlockchainBDB::pop_block(block& blk, std::vector<transaction>& txs) --m_height; } +void BlockchainBDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, std::vector<tx_out_index> &tx_out_indices) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + tx_out_indices.clear(); + + for (const uint64_t &index : global_indices) + { + Dbt_copy<uint32_t> k(index); + Dbt_copy<crypto::hash> v; + + auto get_result = m_output_txs->get(DB_DEFAULT_TX, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = v; + + Dbt_copy<uint64_t> result; + get_result = m_output_indices->get(DB_DEFAULT_TX, &k, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + auto hashindex = tx_out_index(tx_hash, result); + tx_out_indices.push_back(hashindex); + } +} + +void BlockchainBDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + TIME_MEASURE_START(txx); + check_open(); + + bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); + uint64_t max = 0; + for (const uint64_t& index : offsets) + { + if (index > max) + max = index; + } + + // get returned keypairs count +#define DB_COUNT_RECORDS(dbt, cnt) \ + do { \ + uint32_t *_p = (uint32_t *) ((uint8_t *)(dbt)->data + \ + (dbt)->ulen - sizeof(uint32_t)); \ + cnt = 0; \ + while(*_p != (uint32_t) -1) { \ + _p -= 2; \ + ++cnt; \ + } \ + } while(0); \ + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint32_t> v; + uint64_t buflen = 0; + uint64_t t_dbmul = 0; + uint64_t t_dbscan = 0; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + if (max <= 1 && num_elems <= max) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + TIME_MEASURE_START(db2); + if (max <= 1) + { + for (const uint64_t& index : offsets) + { + TIME_MEASURE_START(t_seek); + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + for (uint64_t i = 0; i < index; ++i) + cur->get(&k, &v, DB_NEXT_DUP); + + uint64_t glob_index = v; + + LOG_PRINT_L3("L0->v: " << glob_index); + global_indices.push_back(glob_index); + + TIME_MEASURE_FINISH(t_seek); + } + } + else + { + // setup a 256KB minimum buffer size + uint32_t pagesize = 256 * 1024; + + // Retrieve only a suitable portion of the kvp data, up to somewhere near + // the maximum offset value being retrieved + buflen = (max + 1) * 4 * sizeof(uint64_t); + buflen = ((buflen / pagesize) + ((buflen % pagesize) > 0 ? 1 : 0)) * pagesize; + + bool nomem = false; + Dbt data; + + bool singlebuff = buflen <= m_buffer.get_buffer_size(); + buflen = buflen < m_buffer.get_buffer_size() ? buflen : m_buffer.get_buffer_size(); + bdb_safe_buffer_t::type buffer = nullptr; + bdb_safe_buffer_autolock<bdb_safe_buffer_t> lock(m_buffer, buffer); + + data.set_data(buffer); + data.set_ulen(buflen); + data.set_size(buflen); + data.set_flags(DB_DBT_USERMEM); + + uint32_t curcount = 0; + uint32_t blockstart = 0; + for (const uint64_t& index : offsets) + { + if (index >= num_elems) + { + LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); + break; + } + + // fixme! for whatever reason, the first call to DB_MULTIPLE | DB_SET does not + // retrieve the first value. + if (index <= 1 || nomem) + { + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + + for (uint64_t i = 0; i < index; ++i) + cur->get(&k, &v, DB_NEXT_DUP); + } + else + { + while (index >= curcount) + { + TIME_MEASURE_START(t_db1); + try + { + cur->get(&k, &data, DB_MULTIPLE | (curcount == 0 ? DB_SET : DB_NEXT_DUP)); + blockstart = curcount; + + int count = 0; + DB_COUNT_RECORDS((DBT * ) &data, count); + curcount += count; + } + catch (const std::exception &e) + { + cur.close(); + throw0(DB_ERROR(std::string("Failed on DB_MULTIPLE: ").append(e.what()).c_str())); + } + + TIME_MEASURE_FINISH(t_db1); + t_dbmul += t_db1; + if (singlebuff) + break; + } + + LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); + TIME_MEASURE_START(t_db2); + DBT *pdata = (DBT *) &data; + + uint8_t *value; + uint64_t dlen = 0; + + void *pbase = ((uint8_t *) (pdata->data)) + pdata->ulen - sizeof(uint32_t); + uint32_t *p = (uint32_t *) pbase; + if (*p == (uint32_t) -1) + { + value = NULL; + } + else + { + p -= (index - blockstart) * 2; // index * 4 + 2; <- if DB_MULTIPLE_KEY + value = (uint8_t *) pdata->data + *p--; + dlen = *p--; + if (value == (uint8_t *) pdata->data) + value = NULL; + } + + if (value != NULL) + { + v = dlen == sizeof(uint64_t) ? *((uint64_t *) value) : *((uint32_t *) value); + } + TIME_MEASURE_FINISH(t_db2); + t_dbscan += t_db2; + } + + uint64_t glob_index = v; + + LOG_PRINT_L3("L1->v: " << glob_index); + global_indices.push_back(glob_index); + } + } + TIME_MEASURE_FINISH(db2); + + cur.close(); + + TIME_MEASURE_FINISH(txx); + + LOG_PRINT_L3("blen: " << buflen << " txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan); + +} + +void BlockchainBDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + TIME_MEASURE_START(txx); + outputs.clear(); + + std::vector < uint64_t > global_indices; + get_output_global_indices(amount, offsets, global_indices); + + TIME_MEASURE_START(db3); + if (global_indices.size() > 0) + { + for (const uint64_t &index : global_indices) + { + Dbt_copy<uint32_t> k(index); + Dbt_copy<output_data_t> v; + + auto get_result = m_output_keys->get(DB_DEFAULT_TX, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + output_data_t data = *(output_data_t *) v.get_data(); + outputs.push_back(data); + } + } + + TIME_MEASURE_FINISH(txx); + LOG_PRINT_L3("db3: " << db3); +} + +void BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + std::vector < uint64_t > global_indices; + get_output_global_indices(amount, offsets, global_indices); + + TIME_MEASURE_START(db3); + if (global_indices.size() > 0) + get_output_tx_and_index_from_global(global_indices, indices); + TIME_MEASURE_FINISH(db3); + + LOG_PRINT_L3("db3: " << db3); +} + +void BlockchainBDB::checkpoint_worker() const +{ + LOG_PRINT_L0("Entering BDB checkpoint thread.") + int count = 0; + while(m_run_checkpoint && m_open) + { + // sleep every second, so we don't delay exit condition m_run_checkpoint = false + sleep(1); + // checkpoint every 5 minutes + if(count++ >= 300) + { + count = 0; + if(m_env->txn_checkpoint(0, 0, 0) != 0) + { + LOG_PRINT_L0("BDB txn_checkpoint failed.") + break; + } + } + } + LOG_PRINT_L0("Leaving BDB checkpoint thread.") +} + } // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index fec4ed24f..41f4bcb78 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -30,6 +30,11 @@ #include "blockchain_db/blockchain_db.h" #include "cryptonote_protocol/blobdatatype.h" // for type blobdata +#include <unordered_map> + +// ND: Enables multi-threaded bulk reads for when getting indices. +// TODO: Disabled for now, as it doesn't seem to provide noticeable improvements (??. Reason: TBD. +// #define BDB_BULK_CAN_THREAD namespace cryptonote { @@ -83,10 +88,145 @@ struct bdb_txn_safe { return &m_txn; } - +private: DbTxn* m_txn; }; +// ND: Class to handle buffer management when doing bulk queries +// (DB_MULTIPLE). Allocates buffers then handles thread queuing +// so a fixed set of buffers can be used (instead of allocating +// every time a bulk query is needed). +template <typename T> +class bdb_safe_buffer +{ + // limit the number of buffers to 8 + const size_t MaxAllowedBuffers = 8; +public: + bdb_safe_buffer(size_t num_buffers, size_t count) + { + if(num_buffers > MaxAllowedBuffers) + num_buffers = MaxAllowedBuffers; + + set_count(num_buffers); + for (size_t i = 0; i < num_buffers; i++) + m_buffers.push_back((T) malloc(sizeof(T) * count)); + m_buffer_count = count; + } + + ~bdb_safe_buffer() + { + for (size_t i = 0; i < m_buffers.size(); i++) + { + if (m_buffers[i]) + { + free(m_buffers[i]); + m_buffers[i] = nullptr; + } + } + + m_buffers.resize(0); + } + + T acquire_buffer() + { + std::unique_lock<std::mutex> lock(m_lock); + m_cv.wait(lock, [&]{ return m_count > 0; }); + + --m_count; + size_t index = -1; + for (size_t i = 0; i < m_open_slot.size(); i++) + { + if (m_open_slot[i]) + { + m_open_slot[i] = false; + index = i; + break; + } + } + + assert(index >= 0); + + T buffer = m_buffers[index]; + m_buffer_map.emplace(buffer, index); + return buffer; + } + + void release_buffer(T buffer) + { + std::unique_lock<std::mutex> lock(m_lock); + + assert(buffer != nullptr); + auto it = m_buffer_map.find(buffer); + if (it != m_buffer_map.end()) + { + auto index = it->second; + + assert(index < m_open_slot.size()); + assert(m_open_slot[index] == false); + assert(m_count < m_open_slot.size()); + + ++m_count; + m_open_slot[index] = true; + m_buffer_map.erase(it); + m_cv.notify_one(); + } + } + + size_t get_buffer_size() const + { + return m_buffer_count * sizeof(T); + } + + size_t get_buffer_count() const + { + return m_buffer_count; + } + + typedef T type; + +private: + void set_count(size_t count) + { + assert(count > 0); + m_open_slot.resize(count, true); + m_count = count; + } + + std::vector<T> m_buffers; + std::unordered_map<T, size_t> m_buffer_map; + + std::condition_variable m_cv; + std::vector<bool> m_open_slot; + size_t m_count; + std::mutex m_lock; + + size_t m_buffer_count; +}; + +template <typename T> +class bdb_safe_buffer_autolock +{ +public: + bdb_safe_buffer_autolock(T &safe_buffer, typename T::type &buffer) : + m_safe_buffer(safe_buffer), m_buffer(nullptr) + { + m_buffer = m_safe_buffer.acquire_buffer(); + buffer = m_buffer; + } + + ~bdb_safe_buffer_autolock() + { + if (m_buffer != nullptr) + { + m_safe_buffer.release_buffer(m_buffer); + m_buffer = nullptr; + } + } +private: + T &m_safe_buffer; + typename T::type m_buffer; +}; + class BlockchainBDB : public BlockchainDB { public: @@ -159,8 +299,9 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; - + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); + virtual output_data_t get_output_key(const uint64_t& global_index) const; + virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs); virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; /** @@ -175,9 +316,11 @@ public: tx_out get_output(const uint64_t& index) const; virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, + std::vector<tx_out_index> &tx_out_indices) const; - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; - virtual void get_output_tx_and_index(const uint64_t& amount, std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const; + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); + virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices); virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; @@ -198,7 +341,12 @@ public: virtual void batch_abort(); virtual void pop_block(block& blk, std::vector<transaction>& txs); - virtual bool has_bulk_indices() const { return true; } + +#if defined(BDB_BULK_CAN_THREAD) + virtual bool can_thread_bulk_indices() const { return true; } +#else + virtual bool can_thread_bulk_indices() const { return false; } +#endif private: virtual void add_block( const block& blk @@ -214,7 +362,7 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time); virtual void remove_output(const tx_out& tx_output); @@ -227,6 +375,7 @@ private: virtual void remove_spent_key(const crypto::key_image& k_image); + void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices); /** * @brief convert a tx output to a blob for storage * @@ -253,10 +402,13 @@ private: * * @return the global index of the desired output */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; - + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); + void checkpoint_worker() const; void check_open() const; - void *m_buffer; + bool m_run_checkpoint; + std::unique_ptr<boost::thread> m_checkpoint_thread; + typedef bdb_safe_buffer<void *> bdb_safe_buffer_t; + bdb_safe_buffer_t m_buffer; DbEnv* m_env; diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index bfe93948a..f07f30088 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -64,7 +64,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti { for (uint64_t i = 0; i < tx.vout.size(); ++i) { - add_output(tx_hash, tx.vout[i], i); + add_output(tx_hash, tx.vout[i], i, tx.unlock_time); } for (const txin_v& tx_input : tx.vin) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 7b826f762..ff15109b0 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -138,6 +138,15 @@ namespace cryptonote // typedef for convenience typedef std::pair<crypto::hash, uint64_t> tx_out_index; +#pragma pack(push, 1) +struct output_data_t +{ + crypto::public_key pubkey; + uint64_t unlock_time; + uint64_t height; +}; +#pragma pack(pop) + /*********************************** * Exception Definitions ***********************************/ @@ -279,7 +288,7 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; // tells the subclass to store an output - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0; + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0; // tells the subclass to remove an output virtual void remove_output(const tx_out& tx_output) = 0; @@ -313,7 +322,7 @@ protected: mutable uint64_t time_tx_exists = 0; uint64_t time_commit1 = 0; - + bool m_auto_remove_logs = true; public: @@ -461,7 +470,8 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0; // return public key for output with global output amount <amount> and index <index> - virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0; + virtual output_data_t get_output_key(const uint64_t& global_index) const = 0; // returns the output indexed by <index> in the transaction with hash <h> virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0; @@ -471,9 +481,11 @@ public: // returns the transaction-local reference for the output with <amount> at <index> // return type is pair of tx hash and index - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0; - virtual void get_output_tx_and_index(const uint64_t& amount, std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const = 0; - virtual bool has_bulk_indices() const = 0; + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) = 0; + virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) = 0; + virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) = 0; + + virtual bool can_thread_bulk_indices() const = 0; // return a vector of indices corresponding to the global output index for // each output in the transaction with hash <h> @@ -485,7 +497,10 @@ public: // returns true if key image <img> is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; + void set_auto_remove_logs(bool auto_remove) { m_auto_remove_logs = auto_remove; } + bool m_open; + mutable epee::critical_section m_synchronization_lock; }; // class BlockchainDB diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 0d0b8ba87..22f7dcd49 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -65,10 +65,19 @@ struct lmdb_cur done = false; } - ~lmdb_cur() { close(); } - - operator MDB_cursor*() { return m_cur; } - operator MDB_cursor**() { return &m_cur; } + ~lmdb_cur() + { + close(); + } + + operator MDB_cursor*() + { + return m_cur; + } + operator MDB_cursor**() + { + return &m_cur; + } void close() { @@ -87,7 +96,8 @@ private: template<typename T> struct MDB_val_copy: public MDB_val { - MDB_val_copy(const T &t): t_copy(t) + MDB_val_copy(const T &t) : + t_copy(t) { mv_size = sizeof (T); mv_data = &t_copy; @@ -99,7 +109,8 @@ private: template<> struct MDB_val_copy<cryptonote::blobdata>: public MDB_val { - MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()]) + MDB_val_copy(const cryptonote::blobdata &bd) : + data(new char[bd.size()]) { memcpy(data.get(), bd.data(), bd.size()); mv_size = bd.size(); @@ -109,7 +120,8 @@ private: std::unique_ptr<char[]> data; }; -auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { +auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) +{ const uint64_t va = *(const uint64_t*)a->mv_data; const uint64_t vb = *(const uint64_t*)b->mv_data; if (va < vb) return -1; @@ -117,6 +129,20 @@ auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { else return 1; }; +int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + uint32_t *va = (uint32_t*) a->mv_data; + uint32_t *vb = (uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + const char* const LMDB_BLOCKS = "blocks"; const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; @@ -235,6 +261,26 @@ void mdb_txn_safe::allow_new_txns() void BlockchainLMDB::do_resize(uint64_t increase_size) { + CRITICAL_REGION_LOCAL(m_synchronization_lock); + const uint64_t add_size = 1LL << 30; + + // check disk capacity + try + { + boost::filesystem::path path(m_folder); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < add_size) + { + LOG_PRINT_RED_L0("!! WARNING: Insufficient free space to extend database !!: " << si.available / 1LL << 20L); + return; + } + } + catch(...) + { + // print something but proceed. + LOG_PRINT_YELLOW("Unable to query free disk space.", LOG_LEVEL_0); + } + MDB_envinfo mei; mdb_env_info(m_env, &mei); @@ -250,6 +296,9 @@ void BlockchainLMDB::do_resize(uint64_t increase_size) if (increase_size > 0) new_mapsize = mei.me_mapsize + increase_size; + // add 1Gb per resize, instead of doing a percentage increase + // uint64_t new_mapsize = (double) mei.me_mapsize + add_size; + new_mapsize += (new_mapsize % mst.ms_psize); mdb_txn_safe::prevent_new_txns(); @@ -270,9 +319,7 @@ void BlockchainLMDB::do_resize(uint64_t increase_size) mdb_env_set_mapsize(m_env, new_mapsize); - LOG_PRINT_L0("LMDB Mapsize increased." - << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" - << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); + LOG_PRINT_GREEN("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB", LOG_LEVEL_0); mdb_txn_safe::allow_new_txns(); } @@ -280,6 +327,7 @@ void BlockchainLMDB::do_resize(uint64_t increase_size) // threshold_size is used for batch transactions bool BlockchainLMDB::need_resize(uint64_t threshold_size) const { +#if defined(ENABLE_AUTO_RESIZE) MDB_envinfo mei; mdb_env_info(m_env, &mei); @@ -311,12 +359,19 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const return false; } - if ((double)size_used / mei.me_mapsize > RESIZE_PERCENT) + std::mt19937 engine(std::random_device{}()); + std::uniform_real_distribution<double> fdis(0.6, 0.9); + double resize_percent = fdis(engine); + + if ((double)size_used / mei.me_mapsize > resize_percent) { LOG_PRINT_L1("Threshold met (percent-based)"); return true; } return false; +#else + return false; +#endif } void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks) @@ -389,12 +444,8 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con return threshold_size; } -void BlockchainLMDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const crypto::hash& blk_hash - ) +void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, + const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -545,7 +596,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const } -void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -574,10 +625,15 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou if (tx_output.target.type() == typeid(txout_to_key)) { - MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); - result = mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0); - if (result) - throw0(DB_ERROR(std::string("Failed to add output pubkey to db transaction: ").append(mdb_strerror(result)).c_str())); + output_data_t od; + od.pubkey = boost::get < txout_to_key > (tx_output.target).key; + od.unlock_time = unlock_time; + od.height = m_height; + + MDB_val_copy<output_data_t> data(od); + //MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); + if (mdb_put(*m_write_txn, m_output_keys, &k, &data, 0)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); } @@ -808,47 +864,17 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const return o; } -uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - mdb_txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_output_amounts); - - MDB_val_copy<uint64_t> k(amount); - MDB_val v; - - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) + std::vector <uint64_t> offsets; + std::vector <uint64_t> global_indices; + offsets.push_back(index); + get_output_global_indices(amount, offsets, global_indices); + if (!global_indices.size()) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - - uint64_t glob_index = *(const uint64_t*)v.mv_data; - - cur.close(); - txn.commit(); - - return glob_index; + return global_indices[0]; } void BlockchainLMDB::check_open() const @@ -903,8 +929,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // check for existing LMDB files in base directory boost::filesystem::path old_files = direc.parent_path(); - if (boost::filesystem::exists(old_files / "data.mdb") || - boost::filesystem::exists(old_files / "lock.mdb")) + if (boost::filesystem::exists(old_files / "data.mdb") || boost::filesystem::exists(old_files / "lock.mdb")) { LOG_PRINT_L0("Found existing LMDB files in " << old_files.string()); LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); @@ -970,7 +995,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); - lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); /*************** not used, but kept for posterity @@ -982,6 +1007,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); + mdb_set_compare(txn, m_spent_keys, compare_hash32); + mdb_set_compare(txn, m_block_heights, compare_hash32); + mdb_set_compare(txn, m_txs, compare_hash32); + mdb_set_compare(txn, m_tx_unlocks, compare_hash32); + mdb_set_compare(txn, m_tx_heights, compare_hash32); // get and keep current height MDB_stat db_stats; @@ -995,6 +1025,34 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) throw0(DB_ERROR("Failed to query m_output_indices")); m_num_outputs = db_stats.ms_entries; + // ND: This "new" version of the lmdb database is incompatible with + // the previous version. Ensure that the output_keys database is + // sizeof(output_data_t) in length. Otherwise, inform user and + // terminate. + if(m_height > 0) + { + MDB_val_copy<uint64_t> k(0); + MDB_val v; + auto get_result = mdb_get(txn, m_output_keys, &k, &v); + if(get_result != MDB_SUCCESS) + { + txn.abort(); + m_open = false; + return; + } + + // LOG_PRINT_L0("Output keys size: " << v.mv_size); + if(v.mv_size != sizeof(output_data_t)) + { + txn.abort(); + mdb_env_close(m_env); + m_open = false; + LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version."); + LOG_PRINT_RED_L0("Please delete the existing database and resync."); + return; + } + } + // commit the transaction txn.commit(); @@ -1604,26 +1662,33 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - uint64_t glob_index = get_output_global_index(amount, index); - mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); - MDB_val_copy<uint64_t> k(glob_index); + MDB_val_copy<uint64_t> k(global_index); MDB_val v; auto get_result = mdb_get(txn, m_output_keys, &k, &v); if (get_result == MDB_NOTFOUND) throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + txn.commit(); + return *(output_data_t *) v.mv_data; +} + +output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); - return *(crypto::public_key*)v.mv_data; + uint64_t glob_index = get_output_global_index(amount, index); + return get_output_key(glob_index); } tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const @@ -1731,53 +1796,17 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); } -tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - mdb_txn_safe txn; - mdb_txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - lmdb_cur cur(*txn_ptr, m_output_amounts); - - MDB_val_copy<uint64_t> k(amount); - MDB_val v; - - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) + std::vector < uint64_t > offsets; + std::vector<tx_out_index> indices; + offsets.push_back(index); + get_output_tx_and_index(amount, offsets, indices); + if (!indices.size()) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - - uint64_t glob_index = *(const uint64_t*)v.mv_data; - - cur.close(); - - if (! m_batch_active) - txn.commit(); - - return get_output_tx_and_index_from_global(glob_index); + return indices[0]; } std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const @@ -2016,12 +2045,8 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } -uint64_t BlockchainLMDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector<transaction>& txs - ) +uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, + const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2103,4 +2128,213 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) --m_height; } +void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, + std::vector<tx_out_index> &tx_out_indices) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + tx_out_indices.clear(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + for (const uint64_t &index : global_indices) + { + MDB_val_copy<uint64_t> k(index); + MDB_val v; + + auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = *(crypto::hash*) v.mv_data; + + get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + + auto result = tx_out_index(tx_hash, *(const uint64_t *) v.mv_data); + tx_out_indices.push_back(result); + } + + if (!m_batch_active) + txn.commit(); +} + +void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, + std::vector<uint64_t> &global_indices) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + TIME_MEASURE_START(txx); + check_open(); + global_indices.clear(); + + uint64_t max = 0; + for (const uint64_t &index : offsets) + { + if (index > max) + max = index; + } + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if(m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + lmdb_cur cur(*txn_ptr, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (max <= 1 && num_elems <= max) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + uint64_t t_dbmul = 0; + uint64_t t_dbscan = 0; + if (max <= 1) + { + for (const uint64_t& index : offsets) + { + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + uint64_t glob_index = *(const uint64_t*) v.mv_data; + LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index); + global_indices.push_back(glob_index); + } + } + else + { + uint32_t curcount = 0; + uint32_t blockstart = 0; + for (const uint64_t& index : offsets) + { + if (index >= num_elems) + { + LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); + break; + } + while (index >= curcount) + { + TIME_MEASURE_START(db1); + if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) + { + // allow partial results + result = false; + break; + } + + int count = v.mv_size / sizeof(uint64_t); + + blockstart = curcount; + curcount += count; + TIME_MEASURE_FINISH(db1); + t_dbmul += db1; + } + + LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); + TIME_MEASURE_START(db2); + uint64_t actual_index = index - blockstart; + uint64_t glob_index = ((const uint64_t*) v.mv_data)[actual_index]; + + LOG_PRINT_L3("Amount: " << amount << " M1->v: " << glob_index); + global_indices.push_back(glob_index); + + TIME_MEASURE_FINISH(db2); + t_dbscan += db2; + + } + } + + cur.close(); + if(!m_batch_active) + txn.commit(); + + TIME_MEASURE_FINISH(txx); + LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan); +} + +void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + TIME_MEASURE_START(db3); + check_open(); + outputs.clear(); + + std::vector <uint64_t> global_indices; + get_output_global_indices(amount, offsets, global_indices); + + if (global_indices.size() > 0) + { + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + for (const uint64_t &index : global_indices) + { + MDB_val_copy<uint64_t> k(index); + MDB_val v; + + auto get_result = mdb_get(txn, m_output_keys, &k, &v); + if (get_result != 0) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + output_data_t data = *(output_data_t *) v.mv_data; + outputs.push_back(data); + } + + txn.commit(); + } + + TIME_MEASURE_FINISH(db3); + LOG_PRINT_L3("db3: " << db3); +} + +void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + indices.clear(); + + std::vector <uint64_t> global_indices; + get_output_global_indices(amount, offsets, global_indices); + + TIME_MEASURE_START(db3); + if(global_indices.size() > 0) + { + get_output_tx_and_index_from_global(global_indices, indices); + } + TIME_MEASURE_FINISH(db3); + LOG_PRINT_L3("db3: " << db3); +} + } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index cbab431b9..0facb8713 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -33,6 +33,8 @@ #include <lmdb.h> +#define ENABLE_AUTO_RESIZE + namespace cryptonote { @@ -159,7 +161,9 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); + virtual output_data_t get_output_key(const uint64_t& global_index) const; + virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs); virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; @@ -175,12 +179,12 @@ public: tx_out get_output(const uint64_t& index) const; virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, + std::vector<tx_out_index> &tx_out_indices) const; - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; - virtual void get_output_tx_and_index(const uint64_t& amount, std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const - { - // do nothing - }; + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); + virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices); + virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices); virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; @@ -202,7 +206,7 @@ public: virtual void pop_block(block& blk, std::vector<transaction>& txs); - virtual bool has_bulk_indices() const { return false; } + virtual bool can_thread_bulk_indices() const { return true; } private: void do_resize(uint64_t size_increase=0); @@ -223,7 +227,7 @@ private: virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time); virtual void remove_output(const tx_out& tx_output); @@ -262,7 +266,7 @@ private: * * @return the global index of the desired output */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); void check_open() const; @@ -299,9 +303,18 @@ private: bool m_batch_transactions; // support for batch transactions bool m_batch_active; // whether batch transaction is in progress - constexpr static uint64_t DEFAULT_MAPSIZE = 1 << 30; +#if defined(__arm__) + // force a value so it can compile with 32-bit ARM + constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 31; +#else +#if defined(ENABLE_AUTO_RESIZE) + constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 30; +#else + constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 33; +#endif +#endif + constexpr static float RESIZE_PERCENT = 0.8f; - constexpr static float RESIZE_FACTOR = 1.5f; }; } // namespace cryptonote |