From a74348e1155e6c18b0748795848f56669011cd9b Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 16 Mar 2016 10:24:48 +0000 Subject: Add destructor for readtxns Only if we created the readtxn. Was missing cleanups from exceptions before. --- src/blockchain_db/lmdb/db_lmdb.cpp | 27 ++++++++++++++++++--------- src/blockchain_db/lmdb/db_lmdb.h | 4 +++- 2 files changed, 21 insertions(+), 10 deletions(-) (limited to 'src/blockchain_db/lmdb') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index d2939bf96..9ae7b404b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -228,17 +228,24 @@ mdb_threadinfo::~mdb_threadinfo() mdb_txn_abort(m_ti_rtxn); } -mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) +mdb_txn_safe::mdb_txn_safe(const bool check) : m_txn(NULL), m_tinfo(NULL), m_check(check) { - while (creation_gate.test_and_set()); - num_active_txns++; - creation_gate.clear(); + if (check) + { + while (creation_gate.test_and_set()); + num_active_txns++; + creation_gate.clear(); + } } mdb_txn_safe::~mdb_txn_safe() { LOG_PRINT_L3("mdb_txn_safe: destructor"); - if (m_txn != nullptr) + if (m_tinfo != nullptr) + { + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + } else if (m_txn != nullptr) { if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety { @@ -256,7 +263,8 @@ mdb_txn_safe::~mdb_txn_safe() } mdb_txn_abort(m_txn); } - num_active_txns--; + if (m_check) + num_active_txns--; } void mdb_txn_safe::commit(std::string message) @@ -1303,9 +1311,10 @@ void BlockchainLMDB::unlock() #define TXN_PREFIX_RDONLY() \ MDB_txn *m_txn; \ mdb_txn_cursors *m_cursors; \ - bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); -#define TXN_POSTFIX_RDONLY() \ - if (my_rtxn) block_rtxn_stop() + bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \ + mdb_txn_safe auto_txn(my_rtxn); \ + if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get() +#define TXN_POSTFIX_RDONLY() #define TXN_POSTFIX_SUCCESS() \ do { \ diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 718ee1058..a3f32ffaa 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -115,7 +115,7 @@ typedef struct mdb_threadinfo struct mdb_txn_safe { - mdb_txn_safe(); + mdb_txn_safe(const bool check=true); ~mdb_txn_safe(); void commit(std::string message = ""); @@ -142,8 +142,10 @@ struct mdb_txn_safe static void wait_no_active_txns(); static void allow_new_txns(); + mdb_threadinfo* m_tinfo; MDB_txn* m_txn; bool m_batch_txn = false; + bool m_check; static std::atomic num_active_txns; // could use a mutex here, but this should be sufficient. -- cgit v1.2.3 From db1b2db4d5fd6b4d441faa4bc7c7381e3d88a268 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 19 Mar 2016 12:59:05 +0000 Subject: Reduce log noise --- src/blockchain_db/lmdb/db_lmdb.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/blockchain_db/lmdb') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 9ae7b404b..4e30548fc 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -240,6 +240,8 @@ mdb_txn_safe::mdb_txn_safe(const bool check) : m_txn(NULL), m_tinfo(NULL), m_che mdb_txn_safe::~mdb_txn_safe() { + if (!m_check) + return; LOG_PRINT_L3("mdb_txn_safe: destructor"); if (m_tinfo != nullptr) { @@ -263,8 +265,7 @@ mdb_txn_safe::~mdb_txn_safe() } mdb_txn_abort(m_txn); } - if (m_check) - num_active_txns--; + num_active_txns--; } void mdb_txn_safe::commit(std::string message) -- cgit v1.2.3 From 79117d4275c53ecd472f810e710c74be597758d5 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 20 Mar 2016 18:06:04 +0000 Subject: db_lmdb: include the error codes from lmdb api in error logs --- src/blockchain_db/lmdb/db_lmdb.cpp | 87 ++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 40 deletions(-) (limited to 'src/blockchain_db/lmdb') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 4e30548fc..e928ab803 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -545,11 +545,12 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const { MDB_val_copy parent_key(blk.prev_id); MDB_val parent_h; - if (mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET)) + int result = mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET); + if (result) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); - throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str())); } uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; if (parent_height != m_height - 1) @@ -606,6 +607,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const void BlockchainLMDB::remove_block() { + int result; + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -614,29 +617,29 @@ void BlockchainLMDB::remove_block() MDB_val_copy k(m_height - 1); MDB_val h; - if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) - throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + if ((result = mdb_get(*m_write_txn, m_block_hashes, &k, &h))) + throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str())); - if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block to db transaction")); + if ((result = mdb_del(*m_write_txn, m_blocks, &k, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); - if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + if ((result = mdb_del(*m_write_txn, m_block_sizes, &k, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block size to db transaction: ", result).c_str())); - if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + if ((result = mdb_del(*m_write_txn, m_block_diffs, &k, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block cumulative difficulty to db transaction: ", result).c_str())); - if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + if ((result = mdb_del(*m_write_txn, m_block_coins, &k, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block total generated coins to db transaction: ", result).c_str())); - if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + if ((result = mdb_del(*m_write_txn, m_block_timestamps, &k, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block timestamp to db transaction: ", result).c_str())); - if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) - throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + if ((result = mdb_del(*m_write_txn, m_block_heights, &h, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); + if ((result = mdb_del(*m_write_txn, m_block_hashes, &k, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of block hash to db transaction: ", result).c_str())); } void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) @@ -674,6 +677,8 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { + int result; + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -682,16 +687,16 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) - throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) - throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) - throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + if ((result = mdb_del(*m_write_txn, m_txs, &val_h, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str())); + if ((result = mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx unlock time to db transaction: ", result).c_str())); + if ((result = mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL))) + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx block height to db transaction: ", result).c_str())); remove_tx_outputs(&val_h, tx); - auto result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL); + result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) @@ -741,8 +746,8 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou MDB_val_copy data(od); //MDB_val_copy val_pubkey(boost::get(tx_output.target).key); - if (mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND)) - throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + if ((result = mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND))) + throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str())); } else { @@ -767,7 +772,7 @@ void BlockchainLMDB::remove_tx_outputs(const MDB_val *tx_hash, const transaction } else if (result) { - throw0(DB_ERROR("DB error attempting to get an output")); + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); } else { @@ -879,9 +884,9 @@ void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const MDB { // found the amount output index // now delete it - result = mdb_cursor_del(m_cur_output_amounts, 0); + int result = mdb_cursor_del(m_cur_output_amounts, 0); if (result) - throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); + throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index).append(": ")).c_str(), result).c_str())); } else { @@ -996,6 +1001,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) { + int result; + LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (m_open) @@ -1025,10 +1032,10 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) m_folder = filename; // set up lmdb environment - if (mdb_env_create(&m_env)) - throw0(DB_ERROR("Failed to create lmdb environment")); - if (mdb_env_set_maxdbs(m_env, 20)) - throw0(DB_ERROR("Failed to set max number of dbs")); + if ((result = mdb_env_create(&m_env))) + throw0(DB_ERROR(lmdb_error("Failed to create lmdb environment: ", result).c_str())); + if ((result = mdb_env_set_maxdbs(m_env, 20))) + throw0(DB_ERROR(lmdb_error("Failed to set max number of dbs: ", result).c_str())); size_t mapsize = DEFAULT_MAPSIZE; @@ -1104,14 +1111,14 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // get and keep current height MDB_stat db_stats; - if (mdb_stat(txn, m_blocks, &db_stats)) - throw0(DB_ERROR("Failed to query m_blocks")); + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; // get and keep current number of outputs - if (mdb_stat(txn, m_output_indices, &db_stats)) - throw0(DB_ERROR("Failed to query m_output_indices")); + if ((result = mdb_stat(txn, m_output_indices, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_output_indices: ", result).c_str())); m_num_outputs = db_stats.ms_entries; bool compatible = true; @@ -1232,8 +1239,8 @@ void BlockchainLMDB::reset() check_open(); mdb_txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + if (auto result = mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); mdb_drop(txn, m_blocks, 0); mdb_drop(txn, m_block_timestamps, 0); mdb_drop(txn, m_block_heights, 0); -- cgit v1.2.3 From 600a3cf0c0ca0a99dbdc91d32138db3c8aa4165c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 26 Mar 2016 14:30:23 +0000 Subject: New RPC and daemon command to get output histogram This is a list of existing output amounts along with the number of outputs of that amount in the blockchain. The daemon command takes: - no parameters: all outputs with at least 3 instances - one parameter: all outputs with at least that many instances - two parameters: all outputs within that many instances The default starts at 3 to avoid massive spamming of all dust outputs in the blockchain, and is the current minimum mixin requirement. An optional vector of amounts may be passed, to request histogram only for those outputs. --- src/blockchain_db/lmdb/db_lmdb.cpp | 57 ++++++++++++++++++++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.h | 10 +++++++ 2 files changed, 67 insertions(+) (limited to 'src/blockchain_db/lmdb') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index e928ab803..9b99520a1 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2692,6 +2692,63 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std:: LOG_PRINT_L3("db3: " << db3); } +std::map BlockchainLMDB::get_output_histogram(const std::vector &amounts) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(output_amounts); + + std::map histogram; + MDB_val k; + MDB_val v; + + if (amounts.empty()) + { + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); + op = MDB_NEXT_NODUP; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str())); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); + uint64_t amount = *(const uint64_t*)k.mv_data; + histogram[amount] = num_elems; + } + } + else + { + for (const auto &amount: amounts) + { + MDB_val_copy k(amount); + int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + if (ret == MDB_NOTFOUND) + { + histogram[amount] = 0; + } + else if (ret == MDB_SUCCESS) + { + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); + histogram[amount] = num_elems; + } + else + { + throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str())); + } + } + } + + TXN_POSTFIX_RDONLY(); + + return histogram; +} + void BlockchainLMDB::check_hard_fork_info() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index a3f32ffaa..6cd3e0e8f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -280,6 +280,16 @@ public: virtual void pop_block(block& blk, std::vector& txs); virtual bool can_thread_bulk_indices() const { return true; } + + /** + * @brief return a histogram of outputs on the blockchain + * + * @param amounts optional set of amounts to lookup + * + * @return a set of amount/instances + */ + std::map get_output_histogram(const std::vector &amounts) const; + private: void do_resize(uint64_t size_increase=0); -- cgit v1.2.3