diff options
Diffstat (limited to 'src/blockchain_db/lmdb/db_lmdb.cpp')
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.cpp | 195 |
1 files changed, 172 insertions, 23 deletions
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 201386e47..7d06ae57a 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -148,20 +148,149 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { +std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0}; +std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; -// If m_batch_active is set, a batch transaction exists beyond this class, such -// as a batch import with verification enabled, or possibly (later) a batch -// network sync. -// -// For some of the lookup methods, such as get_block_timestamp(), tx_exists(), -// and get_tx(), when m_batch_active is set, the lookup uses the batch -// transaction. This isn't only because the transaction is available, but it's -// necessary so that lookups include the database updates only present in the -// current batch write. -// -// A regular network sync without batch writes is expected to open a new read -// transaction, as those lookups are part of the validation done prior to the -// write for block and tx data, so no write transaction is open at the time. +mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) +{ + while (creation_gate.test_and_set()); + num_active_txns++; + creation_gate.clear(); +} + +mdb_txn_safe::~mdb_txn_safe() +{ + LOG_PRINT_L3("mdb_txn_safe: destructor"); + if (m_txn != NULL) + { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } + mdb_txn_abort(m_txn); + } + num_active_txns--; +} + +void mdb_txn_safe::commit(std::string message) +{ + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (mdb_txn_commit(m_txn)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; +} + +void mdb_txn_safe::abort() +{ + LOG_PRINT_L3("mdb_txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); + } +} + +uint64_t mdb_txn_safe::num_active_tx() +{ + return num_active_txns; +} + +void mdb_txn_safe::prevent_new_txns() +{ + while (creation_gate.test_and_set()); +} + +void mdb_txn_safe::wait_no_active_txns() +{ + while (num_active_txns > 0); +} + +void mdb_txn_safe::allow_new_txns() +{ + creation_gate.clear(); +} + + + +void BlockchainLMDB::do_resize() +{ + MDB_envinfo mei; + + mdb_env_info(m_env, &mei); + + MDB_stat mst; + + mdb_env_stat(m_env, &mst); + + uint64_t new_mapsize = (double)mei.me_mapsize * RESIZE_FACTOR; + + new_mapsize += (new_mapsize % mst.ms_psize); + + mdb_txn_safe::prevent_new_txns(); + + if (m_write_txn != nullptr) + { + if (m_batch_active) + { + throw0(DB_ERROR("lmdb resizing not yet supported when batch transactions enabled!")); + } + else + { + throw0(DB_ERROR("attempting resize with write transaction in progress, this should not happen!")); + } + } + + mdb_txn_safe::wait_no_active_txns(); + + mdb_env_set_mapsize(m_env, new_mapsize); + + LOG_PRINT_L0("LMDB Mapsize increased." + << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" + << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); + + mdb_txn_safe::allow_new_txns(); +} + +bool BlockchainLMDB::need_resize() +{ + MDB_envinfo mei; + + mdb_env_info(m_env, &mei); + + MDB_stat mst; + + mdb_env_stat(m_env, &mst); + + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + + if ((double)size_used / mei.me_mapsize > 0.8) + { + return true; + } + return false; +} void BlockchainLMDB::add_block( const block& blk , const size_t& block_size @@ -631,6 +760,7 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_batch_transactions = batch_transactions; m_write_txn = nullptr; + m_write_batch_txn = nullptr; m_batch_active = false; m_height = 0; } @@ -672,7 +802,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) if (mdb_env_set_maxdbs(m_env, 20)) throw0(DB_ERROR("Failed to set max number of dbs")); - size_t mapsize = 1LL << 34; + size_t mapsize = DEFAULT_MAPSIZE; if (auto result = mdb_env_set_mapsize(m_env, mapsize)) throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) @@ -1660,16 +1790,21 @@ void BlockchainLMDB::batch_start() throw0(DB_ERROR("batch transactions not enabled")); if (m_batch_active) throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_batch_txn != nullptr) + throw0(DB_ERROR("batch transaction already in progress")); if (m_write_txn) throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); check_open(); + + m_write_batch_txn = new mdb_txn_safe(); + // NOTE: need to make sure it's destroyed properly when done - if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + if (mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); // indicates this transaction is for batch transactions, but not whether it's // active - m_write_batch_txn.m_batch_txn = true; - m_write_txn = &m_write_batch_txn; + m_write_batch_txn->m_batch_txn = true; + m_write_txn = m_write_batch_txn; m_batch_active = true; LOG_PRINT_L3("batch transaction: begin"); } @@ -1681,7 +1816,10 @@ void BlockchainLMDB::batch_commit() throw0(DB_ERROR("batch transactions not enabled")); if (! m_batch_active) throw0(DB_ERROR("batch transaction not in progress")); + if (m_write_batch_txn == nullptr) + throw0(DB_ERROR("batch transaction not in progress")); check_open(); + LOG_PRINT_L3("batch transaction: committing..."); TIME_MEASURE_START(time1); m_write_txn->commit(); @@ -1689,11 +1827,8 @@ void BlockchainLMDB::batch_commit() time_commit1 += time1; LOG_PRINT_L3("batch transaction: committed"); - if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - if (! m_write_batch_txn.m_batch_txn) - throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); - m_write_txn = &m_write_batch_txn; + m_write_txn = nullptr; + delete m_write_batch_txn; } void BlockchainLMDB::batch_stop() @@ -1703,6 +1838,8 @@ void BlockchainLMDB::batch_stop() throw0(DB_ERROR("batch transactions not enabled")); if (! m_batch_active) throw0(DB_ERROR("batch transaction not in progress")); + if (m_write_batch_txn == nullptr) + throw0(DB_ERROR("batch transaction not in progress")); check_open(); LOG_PRINT_L3("batch transaction: committing..."); TIME_MEASURE_START(time1); @@ -1711,6 +1848,8 @@ void BlockchainLMDB::batch_stop() time_commit1 += time1; // for destruction of batch transaction m_write_txn = nullptr; + delete m_write_batch_txn; + m_write_batch_txn = nullptr; m_batch_active = false; LOG_PRINT_L3("batch transaction: end"); } @@ -1726,8 +1865,9 @@ void BlockchainLMDB::batch_abort() // for destruction of batch transaction m_write_txn = nullptr; // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. - m_write_batch_txn.abort(); + m_write_batch_txn->abort(); m_batch_active = false; + m_write_batch_txn = nullptr; LOG_PRINT_L3("batch transaction: aborted"); } @@ -1748,6 +1888,15 @@ uint64_t BlockchainLMDB::add_block( const block& blk LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (m_height % 1000 == 0) + { + if (need_resize()) + { + LOG_PRINT_L0("LMDB memory map needs resized, doing that now."); + do_resize(); + } + } + mdb_txn_safe txn; if (! m_batch_active) { |