diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/blockchain_db/berkeleydb/db_bdb.cpp | 4 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.cpp | 182 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.h | 44 | ||||
| -rw-r--r-- | src/blockchain_utilities/CMakeLists.txt | 10 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_converter.cpp | 4 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_import.cpp | 190 | ||||
| -rw-r--r-- | src/blockchain_utilities/fake_core.h | 26 | ||||
| -rw-r--r-- | src/wallet/wallet2.cpp | 4 |
8 files changed, 362 insertions, 102 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index cdbca52f9..f035bf4c4 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -947,10 +947,12 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) LOG_PRINT_RED_L0("Existing BerkeleyDB database was made by a later version. We don't know how it will change yet."); compatible = false; } - else if (VERSION > 0 && result < VERSION) +#if VERSION > 0 + else if (result < VERSION) { compatible = false; } +#endif } else { diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 0bee8aae9..b43d5742f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -220,6 +220,13 @@ const std::string lmdb_error(const std::string& error_string, int mdb_res) } // anonymous namespace +#define CURSOR(name) \ + if (!m_cur_ ## name) { \ + int result = mdb_cursor_open(*m_write_txn, m_ ## name, &m_cur_ ## name); \ + if (result) \ + throw0(DB_ERROR(std::string("Failed to open cursor: ").append(mdb_strerror(result)).c_str())); \ + } + namespace cryptonote { std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0}; @@ -428,7 +435,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size"); - const uint64_t min_increase_size = 128 * (1 << 20); + const uint64_t min_increase_size = 512 * (1 << 20); uint64_t threshold_size = 0; uint64_t increase_size = 0; if (batch_num_blocks > 0) @@ -464,6 +471,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con // batch size estimate * batch safety factor = final size estimate // Takes into account "reasonable" block size increases in batch. float batch_safety_factor = 1.7f; + float batch_fudge_factor = batch_safety_factor * batch_num_blocks; // estimate of stored block expanded from raw block, including denormalization and db overhead. // Note that this probably doesn't grow linearly with block size. float db_expand_factor = 4.5f; @@ -485,6 +493,13 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con { LOG_PRINT_L1("No existing blocks to check for average block size"); } + else if (m_cum_count) + { + avg_block_size = m_cum_size / m_cum_count; + LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size); + m_cum_size = 0; + m_cum_count = 0; + } else { for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) @@ -502,8 +517,10 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con avg_block_size = min_block_size; LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size); - threshold_size = avg_block_size * db_expand_factor * batch_num_blocks; - threshold_size = threshold_size * batch_safety_factor; + // bigger safety margin on smaller block sizes + if (batch_fudge_factor < 5000.0) + batch_fudge_factor = 5000.0; + threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor; return threshold_size; } @@ -513,16 +530,16 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + CURSOR(block_heights) MDB_val_copy<crypto::hash> val_h(blk_hash); - MDB_val unused; - if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) + if (mdb_cursor_get(m_cur_block_heights, &val_h, NULL, MDB_SET) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); if (m_height > 0) { MDB_val_copy<crypto::hash> parent_key(blk.prev_id); MDB_val parent_h; - if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) + if (mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET)) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); @@ -537,39 +554,48 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const MDB_val_copy<uint64_t> key(m_height); + CURSOR(blocks) + CURSOR(block_sizes) + CURSOR(block_timestamps) + CURSOR(block_diffs) + CURSOR(block_coins) + CURSOR(block_hashes) + MDB_val_copy<blobdata> blob(block_to_blob(blk)); - result = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); + result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<size_t> sz(block_size); - result = mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0); + result = mdb_cursor_put(m_cur_block_sizes, &key, &sz, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add block size to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<uint64_t> ts(blk.timestamp); - result = mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0); + result = mdb_cursor_put(m_cur_block_timestamps, &key, &ts, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add block timestamp to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<difficulty_type> diff(cumulative_difficulty); - result = mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0); + result = mdb_cursor_put(m_cur_block_diffs, &key, &diff, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add block cumulative difficulty to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<uint64_t> coinsgen(coins_generated); - result = mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0); + result = mdb_cursor_put(m_cur_block_coins, &key, &coinsgen, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add block total generated coins to db transaction: ").append(mdb_strerror(result)).c_str())); - result = mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0); + result = mdb_cursor_put(m_cur_block_heights, &val_h, &key, 0); if (result) throw0(DB_ERROR(std::string("Failed to add block height by hash to db transaction: ").append(mdb_strerror(result)).c_str())); - result = mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0); + result = mdb_cursor_put(m_cur_block_hashes, &key, &val_h, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str())); + m_cum_size += block_size; + m_cum_count++; } void BlockchainLMDB::remove_block() @@ -614,23 +640,27 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr int result = 0; + CURSOR(txs) + CURSOR(tx_heights) + CURSOR(tx_unlocks) + MDB_val_copy<crypto::hash> val_h(tx_hash); MDB_val unused; - if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) + if (mdb_cursor_get(m_cur_txs, &val_h, &unused, MDB_SET) == 0) throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); MDB_val_copy<blobdata> blob(tx_to_blob(tx)); - result = mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0); + result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0); if (result) throw0(DB_ERROR(std::string("Failed to add tx blob to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<uint64_t> height(m_height); - result = mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0); + result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0); if (result) throw0(DB_ERROR(std::string("Failed to add tx block height to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<uint64_t> unlock_time(tx.unlock_time); - result = mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0); + result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0); if (result) throw0(DB_ERROR(std::string("Failed to add tx unlock time to db transaction: ").append(mdb_strerror(result)).c_str())); } @@ -668,23 +698,29 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou int result = 0; + CURSOR(output_txs) + CURSOR(tx_outputs) + CURSOR(output_indices) + CURSOR(output_amounts) + CURSOR(output_keys) + MDB_val_copy<uint64_t> k(m_num_outputs); MDB_val_copy<crypto::hash> v(tx_hash); - result = mdb_put(*m_write_txn, m_output_txs, &k, &v, 0); + result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add output tx hash to db transaction: ").append(mdb_strerror(result)).c_str())); - result = mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0); + result = mdb_cursor_put(m_cur_tx_outputs, &v, &k, 0); if (result) throw0(DB_ERROR(std::string("Failed to add <tx hash, global output index> to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<uint64_t> val_local_index(local_index); - result = mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0); + result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND); if (result) throw0(DB_ERROR(std::string("Failed to add tx output index to db transaction: ").append(mdb_strerror(result)).c_str())); MDB_val_copy<uint64_t> val_amount(tx_output.amount); - result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0); + result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &k, 0); if (result) throw0(DB_ERROR(std::string("Failed to add output amount to db transaction: ").append(mdb_strerror(result)).c_str())); @@ -697,7 +733,7 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou MDB_val_copy<output_data_t> data(od); //MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); - if (mdb_put(*m_write_txn, m_output_keys, &k, &data, 0)) + if (mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND)) throw0(DB_ERROR("Failed to add output pubkey to db transaction")); } else @@ -728,7 +764,7 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transa } else { - size_t num_elems = 0; + mdb_size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP); @@ -812,7 +848,7 @@ void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uin else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; + mdb_size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP); @@ -855,15 +891,17 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + CURSOR(spent_keys) + MDB_val_copy<crypto::key_image> val_key(k_image); MDB_val unused; - if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) + if (mdb_cursor_get(m_cur_spent_keys, &val_key, &unused, MDB_SET) == 0) throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); char anything = '\0'; unused.mv_size = sizeof(char); unused.mv_data = &anything; - if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + if (auto result = mdb_cursor_put(m_cur_spent_keys, &val_key, &unused, 0)) throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); } @@ -945,6 +983,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_write_batch_txn = nullptr; m_batch_active = false; m_height = 0; + m_cum_size = 0; + m_cum_count = 0; m_hardfork = nullptr; } @@ -1102,10 +1142,12 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) LOG_PRINT_RED_L0("Existing lmdb database was made by a later version. We don't know how it will change yet."); compatible = false; } - else if (VERSION > 0 && *(const uint32_t*)v.mv_data < VERSION) +#if VERSION > 0 + else if (*(const uint32_t*)v.mv_data < VERSION) { compatible = false; } +#endif } else { @@ -1208,6 +1250,8 @@ void BlockchainLMDB::reset() txn.commit(); m_height = 0; m_num_outputs = 0; + m_cum_size = 0; + m_cum_count = 0; } std::vector<std::string> BlockchainLMDB::get_filenames() const @@ -1733,7 +1777,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const else if (result) throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); - size_t num_elems = 0; + mdb_size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); TXN_POSTFIX_SUCCESS(); @@ -1829,7 +1873,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; + mdb_size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); @@ -1881,7 +1925,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto: else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; + mdb_size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); @@ -2124,6 +2168,8 @@ void BlockchainLMDB::batch_start(uint64_t batch_num_blocks) m_write_batch_txn->m_batch_txn = true; m_write_txn = m_write_batch_txn; m_batch_active = true; + memset(&m_cursors, 0, sizeof(m_cursors)); + LOG_PRINT_L3("batch transaction: begin"); } @@ -2147,6 +2193,7 @@ void BlockchainLMDB::batch_commit() m_write_txn = nullptr; delete m_write_batch_txn; + memset(&m_cursors, 0, sizeof(m_cursors)); } void BlockchainLMDB::batch_stop() @@ -2169,6 +2216,7 @@ void BlockchainLMDB::batch_stop() delete m_write_batch_txn; m_write_batch_txn = nullptr; m_batch_active = false; + memset(&m_cursors, 0, sizeof(m_cursors)); LOG_PRINT_L3("batch transaction: end"); } @@ -2186,6 +2234,7 @@ void BlockchainLMDB::batch_abort() m_write_batch_txn->abort(); m_batch_active = false; m_write_batch_txn = nullptr; + memset(&m_cursors, 0, sizeof(m_cursors)); LOG_PRINT_L3("batch transaction: aborted"); } @@ -2217,6 +2266,7 @@ void BlockchainLMDB::block_txn_start() m_write_txn = nullptr; throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str())); } + memset(&m_cursors, 0, sizeof(m_cursors)); } } @@ -2232,6 +2282,7 @@ void BlockchainLMDB::block_txn_stop() delete m_write_txn; m_write_txn = nullptr; + memset(&m_cursors, 0, sizeof(m_cursors)); } } @@ -2244,6 +2295,7 @@ void BlockchainLMDB::block_txn_abort() { delete m_write_txn; m_write_txn = nullptr; + memset(&m_cursors, 0, sizeof(m_cursors)); } else { @@ -2302,6 +2354,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str())); m_write_txn = &txn; + memset(&m_cursors, 0, sizeof(m_cursors)); } uint64_t num_outputs = m_num_outputs; @@ -2311,7 +2364,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) if (! m_batch_active) { m_write_txn = nullptr; - + memset(&m_cursors, 0, sizeof(m_cursors)); txn.commit(); } } @@ -2319,6 +2372,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) { m_num_outputs = num_outputs; m_write_txn = nullptr; + memset(&m_cursors, 0, sizeof(m_cursors)); throw; } @@ -2387,7 +2441,7 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; + mdb_size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); if (max <= 1 && num_elems <= max) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); @@ -2421,22 +2475,51 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); break; } - while (index >= curcount) + if (!curcount && index > num_elems/2) { - TIME_MEASURE_START(db1); - if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) + mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP); + mdb_cursor_get(cur, &k, &v, MDB_PREV); /* kludge to unset C_EOF */ + mdb_cursor_get(cur, &k, &v, MDB_NEXT); + mdb_cursor_get(cur, &k, &v, MDB_GET_MULTIPLE); + + curcount = num_elems; + while(1) { - // allow partial results - result = false; - break; + TIME_MEASURE_START(db1); + int count = v.mv_size / sizeof(uint64_t); + curcount -= count; + if (curcount > index) + { + mdb_cursor_get(cur, &k, &v, MDB_PREV_MULTIPLE); + } else + { + blockstart = curcount; + curcount += count; + break; + } + TIME_MEASURE_FINISH(db1); + t_dbmul += db1; } - int count = v.mv_size / sizeof(uint64_t); - - blockstart = curcount; - curcount += count; - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; + } else + { + while (index >= curcount) + { + TIME_MEASURE_START(db1); + if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) + { + // allow partial results + result = false; + break; + } + + int count = v.mv_size / sizeof(uint64_t); + + blockstart = curcount; + curcount += count; + TIME_MEASURE_FINISH(db1); + t_dbmul += db1; + } } LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); @@ -2473,13 +2556,14 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui if (global_indices.size() > 0) { TXN_PREFIX_RDONLY(); + lmdb_cur cur(*txn_ptr, m_output_keys); for (const uint64_t &index : global_indices) { MDB_val_copy<uint64_t> k(index); MDB_val v; - auto get_result = mdb_get(*txn_ptr, m_output_keys, &k, &v); + auto get_result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) @@ -2563,7 +2647,7 @@ void BlockchainLMDB::set_hard_fork_starting_height(uint8_t version, uint64_t hei MDB_val_copy<uint8_t> val_key(version); MDB_val_copy<uint64_t> val_value(height); - if (auto result = mdb_put(*txn_ptr, m_hf_starting_heights, &val_key, &val_value, 0)) + if (auto result = mdb_put(*txn_ptr, m_hf_starting_heights, &val_key, &val_value, MDB_APPEND)) throw1(DB_ERROR(std::string("Error adding hard fork starting height to db transaction: ").append(mdb_strerror(result)).c_str())); TXN_BLOCK_POSTFIX_SUCCESS(); @@ -2598,7 +2682,11 @@ void BlockchainLMDB::set_hard_fork_version(uint64_t height, uint8_t version) MDB_val_copy<uint64_t> val_key(height); MDB_val_copy<uint8_t> val_value(version); - if (auto result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0)) + int result; + result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, MDB_APPEND); + if (result == MDB_KEYEXIST) + result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0); + if (result) throw1(DB_ERROR(std::string("Error adding hard fork version to db transaction: ").append(mdb_strerror(result)).c_str())); TXN_BLOCK_POSTFIX_SUCCESS(); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e88bcd01b..150f59475 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -38,6 +38,46 @@ namespace cryptonote { +struct mdb_txn_cursors +{ + MDB_cursor *m_txc_blocks; + MDB_cursor *m_txc_block_heights; + MDB_cursor *m_txc_block_hashes; + MDB_cursor *m_txc_block_timestamps; + MDB_cursor *m_txc_block_sizes; + MDB_cursor *m_txc_block_diffs; + MDB_cursor *m_txc_block_coins; + + MDB_cursor *m_txc_output_txs; + MDB_cursor *m_txc_output_indices; + MDB_cursor *m_txc_output_amounts; + MDB_cursor *m_txc_output_keys; + + MDB_cursor *m_txc_txs; + MDB_cursor *m_txc_tx_heights; + MDB_cursor *m_txc_tx_unlocks; + MDB_cursor *m_txc_tx_outputs; + + MDB_cursor *m_txc_spent_keys; +}; + +#define m_cur_blocks m_cursors.m_txc_blocks +#define m_cur_block_heights m_cursors.m_txc_block_heights +#define m_cur_block_hashes m_cursors.m_txc_block_hashes +#define m_cur_block_timestamps m_cursors.m_txc_block_timestamps +#define m_cur_block_sizes m_cursors.m_txc_block_sizes +#define m_cur_block_diffs m_cursors.m_txc_block_diffs +#define m_cur_block_coins m_cursors.m_txc_block_coins +#define m_cur_output_txs m_cursors.m_txc_output_txs +#define m_cur_output_indices m_cursors.m_txc_output_indices +#define m_cur_output_amounts m_cursors.m_txc_output_amounts +#define m_cur_output_keys m_cursors.m_txc_output_keys +#define m_cur_txs m_cursors.m_txc_txs +#define m_cur_tx_heights m_cursors.m_txc_tx_heights +#define m_cur_tx_unlocks m_cursors.m_txc_tx_unlocks +#define m_cur_tx_outputs m_cursors.m_txc_tx_outputs +#define m_cur_spent_keys m_cursors.m_txc_spent_keys + struct mdb_txn_safe { mdb_txn_safe(); @@ -306,6 +346,8 @@ private: uint64_t m_height; uint64_t m_num_outputs; + mutable uint64_t m_cum_size; // used in batch size estimation + mutable int m_cum_count; std::string m_folder; mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB @@ -313,6 +355,8 @@ private: bool m_batch_transactions; // support for batch transactions bool m_batch_active; // whether batch transaction is in progress + struct mdb_txn_cursors m_cursors; + #if defined(__arm__) // force a value so it can compile with 32-bit ARM constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 31; diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index ad2a5b40a..41c3098a0 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -99,6 +99,11 @@ target_link_libraries(blockchain_converter blockchain_db ${CMAKE_THREAD_LIBS_INIT}) +if(${ARCH_WIDTH} EQUAL 32) + target_compile_definitions(blockchain_converter + PUBLIC -DARCH_WIDTH=32) +endif() + add_dependencies(blockchain_converter version) set_property(TARGET blockchain_converter @@ -117,6 +122,11 @@ target_link_libraries(blockchain_import p2p ${CMAKE_THREAD_LIBS_INIT}) +if(${ARCH_WIDTH} EQUAL 32) + target_compile_definitions(blockchain_import + PUBLIC -DARCH_WIDTH=32) +endif() + add_dependencies(blockchain_import version) set_property(TARGET blockchain_import diff --git a/src/blockchain_utilities/blockchain_converter.cpp b/src/blockchain_utilities/blockchain_converter.cpp index fdd369e79..17b6d81bf 100644 --- a/src/blockchain_utilities/blockchain_converter.cpp +++ b/src/blockchain_utilities/blockchain_converter.cpp @@ -57,12 +57,12 @@ bool opt_testnet = false; // number of blocks per batch transaction // adjustable through command-line argument according to available RAM -#if !defined(WIN32) +#if ARCH_WIDTH != 32 uint64_t db_batch_size_verify = 5000; #else // set a lower default batch size for Windows, pending possible LMDB issue with // large batch size. -uint64_t db_batch_size_verify = 1000; +uint64_t db_batch_size_verify = 100; #endif // converter only uses verify mode diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index d3045a229..67ad24031 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -38,6 +38,7 @@ #include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/json_utils.h" // dump_json() #include "include_base_utils.h" +#include "blockchain_db/db_types.h" #include <lmdb.h> // for db flag arguments @@ -55,11 +56,11 @@ bool opt_testnet = true; // number of blocks per batch transaction // adjustable through command-line argument according to available RAM -#if !defined(WIN32) +#if ARCH_WIDTH != 32 uint64_t db_batch_size = 20000; #else // set a lower default batch size, pending possible LMDB issue with large transaction size -uint64_t db_batch_size = 1000; +uint64_t db_batch_size = 100; #endif // when verifying, use a smaller default batch size so progress is more @@ -77,12 +78,48 @@ using namespace cryptonote; using namespace epee; -int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim) +{ + std::string result; + std::ostringstream s; + std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim)); + result = s.str(); + if (result.length() > 0) + result.erase(result.end()-strlen(delim), result.end()); + return result; +} + +// db_type: lmdb, berkeley +// db_mode: safe, fast, fastest +int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode) +{ + uint64_t BDB_FAST_MODE = 0; + uint64_t BDB_FASTEST_MODE = 0; + uint64_t BDB_SAFE_MODE = 0; + +#if defined(BERKELEY_DB) + BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC; + BDB_FASTEST_MODE = DB_TXN_NOSYNC; + BDB_SAFE_MODE = DB_TXN_SYNC; +#endif + + int db_flags = 0; + bool islmdb = db_type == "lmdb"; + if (db_mode == "safe") + db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE; + else if (db_mode == "fast") + db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE; + else if (db_mode == "fastest") + db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE; + return db_flags; +} + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags) { std::vector<std::string> db_args; boost::split(db_args, db_arg_str, boost::is_any_of("#")); - db_engine = db_args.front(); - boost::algorithm::trim(db_engine); + db_type = db_args.front(); + boost::algorithm::trim(db_type); if (db_args.size() == 1) { @@ -94,28 +131,73 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, in return 1; } +#if !defined(BERKELEY_DB) + if (db_type == "berkeley") + { + LOG_ERROR("BerkeleyDB support disabled."); + return false; + } +#endif + std::string db_arg_str2 = db_args[1]; boost::split(db_args, db_arg_str2, boost::is_any_of(",")); - for (auto& it : db_args) + + // optionally use a composite mode instead of individual flags + const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"}; + std::string db_mode; + if (db_args.size() == 1) { - boost::algorithm::trim(it); - if (it.empty()) - continue; - LOG_PRINT_L1("LMDB flag: " << it); - if (it == "nosync") - mdb_flags |= MDB_NOSYNC; - else if (it == "nometasync") - mdb_flags |= MDB_NOMETASYNC; - else if (it == "writemap") - mdb_flags |= MDB_WRITEMAP; - else if (it == "mapasync") - mdb_flags |= MDB_MAPASYNC; - else if (it == "nordahead") - mdb_flags |= MDB_NORDAHEAD; - else + if (db_modes.count(db_args[0]) > 0) + { + db_mode = db_args[0]; + } + } + if (! db_mode.empty()) + { + db_flags = get_db_flags_from_mode(db_type, db_mode); + } + else + { + for (auto& it : db_args) { - std::cerr << "unrecognized database flag: " << it << ENDL; - return 1; + boost::algorithm::trim(it); + if (it.empty()) + continue; + if (db_type == "lmdb") + { + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + db_flags |= MDB_NOSYNC; + else if (it == "nometasync") + db_flags |= MDB_NOMETASYNC; + else if (it == "writemap") + db_flags |= MDB_WRITEMAP; + else if (it == "mapasync") + db_flags |= MDB_MAPASYNC; + else if (it == "nordahead") + db_flags |= MDB_NORDAHEAD; + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } +#if defined(BERKELEY_DB) + else if (db_type == "berkeley") + { + if (it == "txn_write_nosync") + db_flags = DB_TXN_WRITE_NOSYNC; + else if (it == "txn_nosync") + db_flags = DB_TXN_NOSYNC; + else if (it == "txn_sync") + db_flags = DB_TXN_SYNC; + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } +#endif } } return 0; @@ -131,7 +213,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks) if (simple_core.support_batch) use_batch = true; else - LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring"); } if (use_batch) @@ -176,11 +258,11 @@ template <typename FakeCore> int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0) { #if !defined(BLOCKCHAIN_DB) - static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value, + static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_db, FakeCore>::value, "FakeCore constraint error"); #endif #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) - if (std::is_same<fake_core_lmdb, FakeCore>::value) + if (std::is_same<fake_core_db, FakeCore>::value) { // Reset stats, in case we're using newly created db, accumulating stats // from addition of genesis block. @@ -250,7 +332,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, if (simple_core.support_batch) use_batch = true; else - LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring"); } if (use_batch) @@ -530,11 +612,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, int main(int argc, char* argv[]) { #if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) - std::string default_db_engine = "memory"; + std::string default_db_type = "memory"; + std::string default_db_engine_compiled = "memory"; #else - std::string default_db_engine = "lmdb"; + std::string default_db_type = "lmdb"; + std::string default_db_engine_compiled = "blockchain_db"; #endif + std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types; + db_types_all.insert("memory"); + + std::string available_dbs = join_set_strings(db_types_all, ", "); + available_dbs = "available: " + available_dbs; + uint32_t log_level = LOG_LEVEL_0; uint64_t num_blocks = 0; uint64_t block_stop = 0; @@ -566,8 +656,7 @@ int main(int argc, char* argv[]) , false }; const command_line::arg_descriptor<std::string> arg_database = { - "database", "available: memory, lmdb" - , default_db_engine + "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<bool> arg_verify = {"verify", "Verify blocks and transactions during import", true}; @@ -648,8 +737,6 @@ int main(int argc, char* argv[]) } } - std::vector<std::string> db_engines {"memory", "lmdb"}; - opt_testnet = command_line::get_arg(vm, arg_testnet_on); auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; m_config_folder = command_line::get_arg(vm, data_dir_arg); @@ -677,23 +764,34 @@ int main(int argc, char* argv[]) } - std::string db_engine; - int mdb_flags = 0; + std::string db_type; + std::string db_engine_compiled; + int db_flags = 0; int res = 0; - res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + res = parse_db_arguments(db_arg_str, db_type, db_flags); if (res) { std::cerr << "Error parsing database argument(s)" << ENDL; return 1; } - if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) + if (db_types_all.count(db_type) == 0) { - std::cerr << "Invalid database engine: " << db_engine << std::endl; + std::cerr << "Invalid database type: " << db_type << std::endl; return 1; } - LOG_PRINT_L0("database: " << db_engine); + if ((db_type == "lmdb") || (db_type == "berkeley")) + { + db_engine_compiled = "blockchain_db"; + } + else + { + db_engine_compiled = "memory"; + } + + LOG_PRINT_L0("database: " << db_type); + LOG_PRINT_L0("database flags: " << db_flags); LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); if (opt_batch) { @@ -722,31 +820,31 @@ int main(int argc, char* argv[]) // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - if (db_engine == "lmdb") + if (db_type == "lmdb" || db_type == "berkeley") { - fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); + fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags); import_from_file(simple_core, import_file_path, block_stop); } - else if (db_engine == "memory") + else if (db_type == "memory") { fake_core_memory simple_core(m_config_folder, opt_testnet); import_from_file(simple_core, import_file_path, block_stop); } else { - std::cerr << "database engine unrecognized" << ENDL; + std::cerr << "database type unrecognized" << ENDL; return 1; } // for multi_db_compile: #else - if (db_engine != default_db_engine) + if (db_engine_compiled != default_db_engine_compiled) { - std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; + std::cerr << "Invalid database type for compiled version: " << db_type << std::endl; return 1; } #if BLOCKCHAIN_DB == DB_LMDB - fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); + fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags); #else fake_core_memory simple_core(m_config_folder, opt_testnet); #endif diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h index 29f34026d..2fb5031d3 100644 --- a/src/blockchain_utilities/fake_core.h +++ b/src/blockchain_utilities/fake_core.h @@ -33,6 +33,9 @@ #include "cryptonote_core/blockchain_storage.h" // in-memory DB #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" +#if defined(BERKELEY_DB) +#include "blockchain_db/berkeleydb/db_bdb.h" +#endif using namespace cryptonote; @@ -47,7 +50,7 @@ namespace #if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB -struct fake_core_lmdb +struct fake_core_db { Blockchain m_storage; HardFork* m_hardfork = nullptr; @@ -57,15 +60,26 @@ struct fake_core_lmdb // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) + fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(&m_storage), m_storage(m_pool) // for multi_db_compile: #else - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) + fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool) #endif { m_pool.init(path.string()); - BlockchainDB* db = new BlockchainLMDB(); + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + db = new BlockchainLMDB(); +#if defined(BERKELEY_DB) + else if (db_type == "berkeley") + db = new BlockchainBDB(); +#endif + else + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } boost::filesystem::path folder(path); @@ -76,7 +90,7 @@ struct fake_core_lmdb const std::string filename = folder.string(); try { - db->open(filename, mdb_flags); + db->open(filename, db_flags); } catch (const std::exception& e) { @@ -96,7 +110,7 @@ struct fake_core_lmdb support_batch = true; support_add_block = true; } - ~fake_core_lmdb() + ~fake_core_db() { m_storage.get_db().check_hard_fork_info(); m_storage.deinit(); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index ed6d3842d..ce829b00f 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -902,6 +902,10 @@ bool wallet2::clear() m_blockchain.clear(); m_transfers.clear(); m_key_images.clear(); + m_unconfirmed_txs.clear(); + m_payments.clear(); + m_tx_keys.clear(); + m_confirmed_txs.clear(); m_local_bc_height = 1; return true; } |