diff options
Diffstat (limited to 'src/blockchain_db/lmdb/db_lmdb.cpp')
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.cpp | 622 |
1 files changed, 590 insertions, 32 deletions
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 2b5fa7fd9..9d2f7821e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -29,12 +29,14 @@ #include <boost/filesystem.hpp> #include <boost/format.hpp> +#include <boost/circular_buffer.hpp> #include <memory> // std::unique_ptr #include <cstring> // memcpy #include "string_tools.h" #include "file_io_utils.h" #include "common/util.h" +#include "common/pruning.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "crypto/crypto.h" #include "profile_tools.h" @@ -52,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 3 +#define VERSION 4 namespace { @@ -130,14 +132,20 @@ private: std::unique_ptr<char[]> data; }; -int compare_uint64(const MDB_val *a, const MDB_val *b) +} + +namespace cryptonote { - const uint64_t va = *(const uint64_t *)a->mv_data; - const uint64_t vb = *(const uint64_t *)b->mv_data; + +int BlockchainLMDB::compare_uint64(const MDB_val *a, const MDB_val *b) +{ + uint64_t va, vb; + memcpy(&va, a->mv_data, sizeof(va)); + memcpy(&vb, b->mv_data, sizeof(vb)); return (va < vb) ? -1 : va > vb; } -int compare_hash32(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_hash32(const MDB_val *a, const MDB_val *b) { uint32_t *va = (uint32_t*) a->mv_data; uint32_t *vb = (uint32_t*) b->mv_data; @@ -151,13 +159,18 @@ int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } -int compare_string(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b) { const char *va = (const char*) a->mv_data; const char *vb = (const char*) b->mv_data; return strcmp(va, vb); } +} + +namespace +{ + /* DB schema: * * Table Key Data @@ -169,6 +182,7 @@ int compare_string(const MDB_val *a, const MDB_val *b) * txs_pruned txn ID pruned txn blob * txs_prunable txn ID prunable txn blob * txs_prunable_hash txn ID prunable txn hash + * txs_prunable_tip txn ID height * tx_indices txn hash {txn ID, metadata} * tx_outputs txn ID [txn amount output indices] * @@ -196,6 +210,7 @@ const char* const LMDB_TXS = "txs"; const char* const LMDB_TXS_PRUNED = "txs_pruned"; const char* const LMDB_TXS_PRUNABLE = "txs_prunable"; const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash"; +const char* const LMDB_TXS_PRUNABLE_TIP = "txs_prunable_tip"; const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -263,7 +278,7 @@ typedef struct mdb_block_info_old crypto::hash bi_hash; } mdb_block_info_old; -typedef struct mdb_block_info +typedef struct mdb_block_info_2 { uint64_t bi_height; uint64_t bi_timestamp; @@ -272,18 +287,27 @@ typedef struct mdb_block_info difficulty_type bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; -} mdb_block_info; +} mdb_block_info_2; + +typedef struct mdb_block_info_3 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; + uint64_t bi_cum_rct; + uint64_t bi_long_term_block_weight; +} mdb_block_info_3; + +typedef mdb_block_info_3 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; uint64_t bh_height; } blk_height; -typedef struct txindex { - crypto::hash key; - tx_data_t data; -} txindex; - typedef struct pre_rct_outkey { uint64_t amount_index; uint64_t output_id; @@ -495,7 +519,7 @@ void BlockchainLMDB::do_resize(uint64_t increase_size) mdb_env_stat(m_env, &mst); // add 1Gb per resize, instead of doing a percentage increase - uint64_t new_mapsize = (double) mei.me_mapsize + add_size; + uint64_t new_mapsize = (uint64_t) mei.me_mapsize + add_size; // If given, use increase_size instead of above way of resizing. // This is currently used for increasing by an estimated size at start of new @@ -549,18 +573,18 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const // additional size needed. uint64_t size_used = mst.ms_psize * mei.me_last_pgno; - LOG_PRINT_L1("DB map size: " << mei.me_mapsize); - LOG_PRINT_L1("Space used: " << size_used); - LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); - LOG_PRINT_L1("Size threshold: " << threshold_size); + MDEBUG("DB map size: " << mei.me_mapsize); + MDEBUG("Space used: " << size_used); + MDEBUG("Space remaining: " << mei.me_mapsize - size_used); + MDEBUG("Size threshold: " << threshold_size); float resize_percent = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); + MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { if (mei.me_mapsize - size_used < threshold_size) { - LOG_PRINT_L1("Threshold met (size-based)"); + MINFO("Threshold met (size-based)"); return true; } else @@ -569,7 +593,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const if ((double)size_used / mei.me_mapsize > resize_percent) { - LOG_PRINT_L1("Threshold met (percent-based)"); + MINFO("Threshold met (percent-based)"); return true; } return false; @@ -581,7 +605,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size"); + MTRACE("[" << __func__ << "] " << "checking DB size"); const uint64_t min_increase_size = 512 * (1 << 20); uint64_t threshold_size = 0; uint64_t increase_size = 0; @@ -685,7 +709,7 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -745,6 +769,7 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const diff const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; bi.bi_cum_rct += bi_prev->bi_cum_rct; } + bi.bi_long_term_block_weight = long_term_block_weight; MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); @@ -811,6 +836,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_indices) MDB_val_set(val_tx_id, tx_id); @@ -858,6 +884,14 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); + if (get_blockchain_pruning_seed()) + { + MDB_val_set(val_height, m_height); + result = mdb_cursor_put(m_cur_txs_prunable_tip, &val_tx_id, &val_height, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str())); + } + if (tx.version > 1) { MDB_val_set(val_prunable_hash, tx_prunable_hash); @@ -883,6 +917,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_outputs) MDB_val_set(val_h, tx_hash); @@ -898,11 +933,25 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET))) + result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + } + else if (result != MDB_NOTFOUND) throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str())); - result = mdb_cursor_del(m_cur_txs_prunable, 0); - if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + + result = mdb_cursor_get(m_cur_txs_prunable_tip, &val_tx_id, NULL, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Failed to locate tx id for removal: ", result).c_str())); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable_tip, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str())); + } if (tx.version > 1) { @@ -1280,14 +1329,14 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) MDB_envinfo mei; mdb_env_info(m_env, &mei); - uint64_t cur_mapsize = (double)mei.me_mapsize; + uint64_t cur_mapsize = (uint64_t)mei.me_mapsize; if (cur_mapsize < mapsize) { if (auto result = mdb_env_set_mapsize(m_env, mapsize)) throw0(DB_ERROR(lmdb_error("Failed to set max memory map size: ", result).c_str())); mdb_env_info(m_env, &mei); - cur_mapsize = (double)mei.me_mapsize; + cur_mapsize = (uint64_t)mei.me_mapsize; LOG_PRINT_L1("LMDB memory map size: " << cur_mapsize); } @@ -1308,6 +1357,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // open necessary databases, and set properties as needed // uses macros to avoid having to change things too many places + // also change blockchain_prune.cpp to match lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); @@ -1316,7 +1366,9 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned"); lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable"); - lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + if (!(mdb_flags & MDB_RDONLY)) + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_TIP, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_tip, "Failed to open db handle for m_txs_prunable_tip"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); @@ -1344,6 +1396,10 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_output_txs, compare_uint64); mdb_set_dupsort(txn, m_block_info, compare_uint64); + if (!(mdb_flags & MDB_RDONLY)) + mdb_set_dupsort(txn, m_txs_prunable_tip, compare_uint64); + mdb_set_compare(txn, m_txs_prunable, compare_uint64); + mdb_set_dupsort(txn, m_txs_prunable_hash, compare_uint64); mdb_set_compare(txn, m_txpool_meta, compare_hash32); mdb_set_compare(txn, m_txpool_blob, compare_hash32); @@ -1502,6 +1558,8 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str())); if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_prunable_tip, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_tip: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_indices, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_outputs, 0)) @@ -1827,6 +1885,290 @@ cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid return bd; } +uint32_t BlockchainLMDB::get_blockchain_pruning_seed() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(properties) + MDB_val_str(k, "pruning_seed"); + MDB_val v; + int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return 0; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning seed: ", result).c_str())); + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + uint32_t pruning_seed; + memcpy(&pruning_seed, v.mv_data, sizeof(pruning_seed)); + TXN_POSTFIX_RDONLY(); + return pruning_seed; +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to find transaction pruned data: ", ret).c_str())); + if (v.mv_size == 0) + throw0(DB_ERROR("Invalid transaction pruned data")); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +enum { prune_mode_prune, prune_mode_update, prune_mode_check }; + +bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + const uint32_t log_stripes = tools::get_pruning_log_stripes(pruning_seed); + if (log_stripes && log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Pruning seed not in range")); + pruning_seed = tools::get_pruning_stripe(pruning_seed);; + if (pruning_seed > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + throw0(DB_ERROR("Pruning seed not in range")); + check_open(); + + TIME_MEASURE_START(t); + + size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0; + uint64_t n_bytes = 0; + + mdb_txn_safe txn; + auto result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages0 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + + MDB_val_str(k, "pruning_seed"); + MDB_val v; + result = mdb_get(txn, m_properties, &k, &v); + bool prune_tip_table = false; + if (result == MDB_NOTFOUND) + { + // not pruned yet + if (mode != prune_mode_prune) + { + txn.abort(); + TIME_MEASURE_FINISH(t); + MDEBUG("Pruning not enabled, nothing to do"); + return true; + } + if (pruning_seed == 0) + pruning_seed = tools::get_random_stripe(); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + v.mv_data = &pruning_seed; + v.mv_size = sizeof(pruning_seed); + result = mdb_put(txn, m_properties, &k, &v, 0); + if (result) + throw0(DB_ERROR("Failed to save pruning seed")); + prune_tip_table = false; + } + else if (result == 0) + { + // pruned already + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + const uint32_t data = *(const uint32_t*)v.mv_data; + if (pruning_seed == 0) + pruning_seed = tools::get_pruning_stripe(data); + if (tools::get_pruning_stripe(data) != pruning_seed) + throw0(DB_ERROR("Blockchain already pruned with different seed")); + if (tools::get_pruning_log_stripes(data) != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Blockchain already pruned with different base")); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + prune_tip_table = (mode == prune_mode_update); + } + else + { + throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str())); + } + + if (mode == prune_mode_check) + MINFO("Checking blockchain pruning..."); + else + MINFO("Pruning blockchain..."); + + MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip; + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + const uint64_t blockchain_height = height(); + + if (prune_tip_table) + { + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_txs_prunable_tip, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + uint64_t block_height; + memcpy(&block_height, v.mv_data, sizeof(block_height)); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS < blockchain_height) + { + ++n_total_records; + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &k)) + { + ++n_prunable_records; + result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str())); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += k.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + result = mdb_cursor_del(c_txs_prunable_tip, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str())); + } + } + } + else + { + MDB_cursor *c_tx_indices; + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + ++n_total_records; + //const txindex *ti = (const txindex *)v.mv_data; + txindex ti; + memcpy(&ti, v.mv_data, sizeof(ti)); + const uint64_t block_height = ti.data.block_id; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDB_val_set(kp, ti.data.tx_id); + MDB_val_set(vp, block_height); + if (mode == prune_mode_check) + { + result = mdb_cursor_get(c_txs_prunable_tip, &kp, &vp, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Transaction not found in prunable tip table for height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + result = mdb_cursor_put(c_txs_prunable_tip, &kp, &vp, 0); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + } + } + MDB_val_set(kp, ti.data.tx_id); + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &kp)) + { + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (mode == prune_mode_check) + { + if (result != MDB_NOTFOUND) + MERROR("Prunable data found for pruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + ++n_prunable_records; + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += kp.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + } + else + { + if (mode == prune_mode_check) + { + MDB_val_set(kp, ti.data.tx_id); + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Prunable data not found for unpruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + } + } + mdb_cursor_close(c_tx_indices); + } + + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages1 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + const size_t db_bytes = (pages0 - pages1) * db_stats.ms_psize; + + mdb_cursor_close(c_txs_prunable_tip); + mdb_cursor_close(c_txs_prunable); + mdb_cursor_close(c_txs_pruned); + + txn.commit(); + + TIME_MEASURE_FINISH(t); + + MINFO((mode == prune_mode_check ? "Checked" : "Pruned") << " blockchain in " << + t << " ms: " << (n_bytes/1024.0f/1024.0f) << " MB (" << db_bytes/1024.0f/1024.0f << " MB) pruned in " << + n_pruned_records << " records (" << pages0 - pages1 << "/" << pages0 << " " << db_stats.ms_psize << " byte pages), " << + n_prunable_records << "/" << n_total_records << " pruned records"); + return true; +} + +bool BlockchainLMDB::prune_blockchain(uint32_t pruning_seed) +{ + return prune_worker(prune_mode_prune, pruning_seed); +} + +bool BlockchainLMDB::update_pruning() +{ + return prune_worker(prune_mode_update, 0); +} + +bool BlockchainLMDB::check_pruning() +{ + return prune_worker(prune_mode_check, 0); +} + bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2160,6 +2502,29 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh return ret; } +uint64_t BlockchainLMDB::get_block_long_term_weight(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get block long term weight from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block info not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a long term block weight from the db")); + + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_long_term_block_weight; + TXN_POSTFIX_RDONLY(); + return ret; +} + crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2428,6 +2793,36 @@ bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobd return true; } +bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_prunable); + + MDB_val_set(v, h); + MDB_val result; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + const txindex *tip = (const txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result, MDB_SET); + } + if (get_result == MDB_NOTFOUND) + return false; + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + TXN_POSTFIX_RDONLY(); + + return true; +} + bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3168,7 +3563,7 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3187,7 +3582,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const try { - BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -4392,6 +4787,7 @@ void BlockchainLMDB::migrate_2_3() throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); mdb_set_dupsort(txn, m_block_info, compare_uint64); @@ -4412,6 +4808,166 @@ void BlockchainLMDB::migrate_2_3() txn.commit(); } +void BlockchainLMDB::migrate_3_4() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + bool past_long_term_weight = false; + + MGINFO_YELLOW("Migrating blockchain from DB version 3 to 4 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + boost::circular_buffer<uint64_t> long_term_block_weights(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE); + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_2 *bi_old = (const mdb_block_info_2*)v.mv_data; + mdb_block_info_3 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + + // get block major version to determine which rule is in place + if (!past_long_term_weight) + { + MDB_val_copy<uint64_t> kb(bi.bi_height); + MDB_val vb; + result = mdb_cursor_get(c_blocks, &kb, &vb, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + if (vb.mv_size == 0) + throw0(DB_ERROR("Invalid data from m_blocks")); + const uint8_t block_major_version = *((const uint8_t*)vb.mv_data); + if (block_major_version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + past_long_term_weight = true; + } + + uint64_t long_term_block_weight; + if (past_long_term_weight) + { + std::vector<uint64_t> weights(long_term_block_weights.begin(), long_term_block_weights.end()); + uint64_t long_term_effective_block_median_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, epee::misc_utils::median(weights)); + long_term_block_weight = std::min<uint64_t>(bi.bi_weight, long_term_effective_block_median_weight + long_term_effective_block_median_weight * 2 / 5); + } + else + { + long_term_block_weight = bi.bi_weight; + } + long_term_block_weights.push_back(long_term_block_weight); + bi.bi_long_term_block_weight = long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 4; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { if (oldversion < 1) @@ -4420,6 +4976,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_1_2(); if (oldversion < 3) migrate_2_3(); + if (oldversion < 4) + migrate_3_4(); } } // namespace cryptonote |