aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--external/db_drivers/liblmdb/mdb.c2
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.cpp728
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.h26
3 files changed, 264 insertions, 492 deletions
diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c
index a366b4024..acab07b82 100644
--- a/external/db_drivers/liblmdb/mdb.c
+++ b/external/db_drivers/liblmdb/mdb.c
@@ -6771,8 +6771,8 @@ set1:
if (op == MDB_GET_BOTH || rc > 0)
return MDB_NOTFOUND;
rc = 0;
- *data = olddata;
}
+ *data = olddata;
} else {
if (mc->mc_xcursor)
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index cd35286c5..604fde0aa 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -46,7 +46,7 @@ using epee::string_tools::pod_to_hex;
// Increase when the DB changes in a non backward compatible way, and there
// is no automatic conversion, so that a full resync is needed.
-#define VERSION 0
+#define VERSION 1
namespace
{
@@ -65,6 +65,8 @@ inline void throw1(const T &e)
throw e;
}
+#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val}
+
template<typename T>
struct MDB_val_copy: public MDB_val
{
@@ -152,9 +154,7 @@ const char* const LMDB_TX_INDICES = "tx_indices";
const char* const LMDB_TX_OUTPUTS = "tx_outputs";
const char* const LMDB_OUTPUT_TXS = "output_txs";
-const char* const LMDB_OUTPUT_INDICES = "output_indices";
const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts";
-const char* const LMDB_OUTPUT_KEYS = "output_keys";
const char* const LMDB_SPENT_KEYS = "spent_keys";
const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights";
@@ -192,10 +192,12 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi
int result = mdb_cursor_open(m_txn, m_ ## name, (MDB_cursor **)&m_cur_ ## name); \
if (result) \
throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \
- if (!m_write_txn) \
+ if (m_cursors != &m_wcursors) \
m_tinfo->m_ti_rflags.m_rf_ ## name = true; \
- } else if (!m_write_txn && !m_tinfo->m_ti_rflags.m_rf_ ## name) { \
- mdb_cursor_renew(m_txn, m_cur_ ## name); \
+ } else if (m_cursors != &m_wcursors && !m_tinfo->m_ti_rflags.m_rf_ ## name) { \
+ int result = mdb_cursor_renew(m_txn, m_cur_ ## name); \
+ if (result) \
+ throw0(DB_ERROR(lmdb_error("Failed to renew cursor: ", result).c_str())); \
m_tinfo->m_ti_rflags.m_rf_ ## name = true; \
}
@@ -223,10 +225,16 @@ typedef struct txindex {
} txindex;
typedef struct outkey {
- uint64_t num_outputs;
+ uint64_t amount_index;
+ uint64_t tx_index;
output_data_t data;
} outkey;
+typedef struct outtx {
+ crypto::hash tx_hash;
+ uint64_t local_index;
+} outtx;
+
std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;
@@ -241,17 +249,26 @@ mdb_threadinfo::~mdb_threadinfo()
mdb_txn_abort(m_ti_rtxn);
}
-mdb_txn_safe::mdb_txn_safe() : m_txn(NULL)
+mdb_txn_safe::mdb_txn_safe(const bool check) : m_txn(NULL), m_tinfo(NULL), m_check(check)
{
- while (creation_gate.test_and_set());
- num_active_txns++;
- creation_gate.clear();
+ if (check)
+ {
+ while (creation_gate.test_and_set());
+ num_active_txns++;
+ creation_gate.clear();
+ }
}
mdb_txn_safe::~mdb_txn_safe()
{
+ if (!m_check)
+ return;
LOG_PRINT_L3("mdb_txn_safe: destructor");
- if (m_txn != nullptr)
+ if (m_tinfo != nullptr)
+ {
+ mdb_txn_reset(m_tinfo->m_ti_rtxn);
+ memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
+ } else if (m_txn != nullptr)
{
if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
{
@@ -550,11 +567,12 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
{
blk_height ph = {blk.prev_id, 0};
MDB_val parent_key = {sizeof(ph), (void *)&ph};
- if (mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH))
+ int result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &parent_key, MDB_GET_BOTH);
+ if (result)
{
LOG_PRINT_L3("m_height: " << m_height);
LOG_PRINT_L3("parent_key: " << blk.prev_id);
- throw0(DB_ERROR("Failed to get top block hash to check for new block's parent"));
+ throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str()));
}
blk_height *prev = (blk_height *)parent_key.mv_data;
if (prev->height != m_height - 1)
@@ -595,6 +613,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
void BlockchainLMDB::remove_block()
{
+ int result;
+
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@@ -605,22 +625,22 @@ void BlockchainLMDB::remove_block()
CURSOR(block_info)
MDB_val_copy<uint64_t> k(m_height - 1);
MDB_val h;
- if (mdb_cursor_get(m_cur_block_info, &k, &h, MDB_SET))
- throw1(BLOCK_DNE("Attempting to remove block that's not in the db"));
+ if ((result = mdb_cursor_get(m_cur_block_info, &k, &h, MDB_SET)))
+ throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str()));
// must use h now; deleting from m_block_info will invalidate it
mdb_block_info *bi = (mdb_block_info *)h.mv_data;
blk_height bh = {bi->bi_hash, 0};
h.mv_data = (void *)&bh;
h.mv_size = sizeof(bh);
- if (mdb_del(*m_write_txn, m_block_heights, (MDB_val *)&zerokval, &h))
- throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction"));
+ if ((result = mdb_del(*m_write_txn, m_block_heights, (MDB_val *)&zerokval, &h)))
+ throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str()));
- if (mdb_del(*m_write_txn, m_blocks, &k, NULL))
- throw1(DB_ERROR("Failed to add removal of block to db transaction"));
+ if ((result = mdb_del(*m_write_txn, m_blocks, &k, NULL)))
+ throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str()));
- if (mdb_cursor_del(m_cur_block_info, 0))
- throw1(DB_ERROR("Failed to add removal of block info to db transaction"));
+ if ((result = mdb_cursor_del(m_cur_block_info, 0)))
+ throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str()));
}
uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash)
@@ -670,6 +690,8 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
// passing it in to functions like this
void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx)
{
+ int result;
+
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@@ -678,7 +700,6 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
txindex ti = {tx_hash};
MDB_val val_h = {sizeof(ti), (void *)&ti};
- MDB_val v;
if (mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH))
throw1(TX_DNE("Attempting to remove transaction that isn't in the db"));
@@ -686,12 +707,12 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
uint64_t tx_index = tip->data.tx_index;
MDB_val_copy<uint64_t> val_tx_index(tx_index);
- if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL))
- throw1(DB_ERROR("Failed to add removal of tx to db transaction"));
+ if ((result = mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL)))
+ throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str()));
remove_tx_outputs(tx_index, tx);
- int result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL);
+ result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL);
if (result == MDB_NOTFOUND)
LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash);
else if (result)
@@ -708,6 +729,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
m_num_txs--;
}
+// global_output_index is no longer used for locating outputs
void BlockchainLMDB::add_output(const crypto::hash& tx_hash,
const tx_out& tx_output,
const uint64_t& local_index,
@@ -722,55 +744,53 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash,
int result = 0;
CURSOR(output_txs)
- CURSOR(output_indices)
CURSOR(output_amounts)
- CURSOR(output_keys)
- MDB_val_copy<uint64_t> k(m_num_outputs);
- MDB_val_copy<crypto::hash> v(tx_hash);
+ if (tx_output.target.type() != typeid(txout_to_key))
+ throw0(DB_ERROR("Wrong output type: expected txout_to_key"));
- result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND);
- if (result)
- throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str()));
+ MDB_val_set(k, m_num_outputs);
- MDB_val_copy<uint64_t> val_local_index(local_index);
- result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND);
- if (result)
- throw0(DB_ERROR(lmdb_error("Failed to add tx output index to db transaction: ", result).c_str()));
-
- MDB_val_copy<uint64_t> val_amount(tx_output.amount);
- result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &k, 0);
- if (result)
- throw0(DB_ERROR(lmdb_error("Failed to add output amount to db transaction: ", result).c_str()));
+ outtx ot = {tx_hash, local_index};
+ MDB_val_set(vot, ot);
- size_t num_elems = 0;
- result = mdb_cursor_count(m_cur_output_amounts, &num_elems);
+ result = mdb_cursor_put(m_cur_output_txs, &k, &vot, MDB_APPEND);
if (result)
- throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str()));
-
- amount_output_index = num_elems - 1;
- global_output_index = m_num_outputs;
-
- if (tx_output.target.type() == typeid(txout_to_key))
- {
- outkey ok;
- ok.num_outputs = m_num_outputs;
- ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key;
- ok.data.unlock_time = unlock_time;
- ok.data.height = m_height;
+ throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str()));
- MDB_val data = {sizeof(ok), (void *)&ok};
- if (mdb_cursor_put(m_cur_output_keys, (MDB_val *)&zerokval, &data, MDB_APPENDDUP))
- throw0(DB_ERROR("Failed to add output pubkey to db transaction"));
- }
+ outkey ok;
+ MDB_val data;
+ MDB_val_copy<uint64_t> val_amount(tx_output.amount);
+ result = mdb_cursor_get(m_cur_output_amounts, &val_amount, &data, MDB_SET);
+ if (!result)
+ {
+ mdb_size_t num_elems = 0;
+ result = mdb_cursor_count(m_cur_output_amounts, &num_elems);
+ if (result)
+ throw0(DB_ERROR(std::string("Failed to get number of outputs for amount: ").append(mdb_strerror(result)).c_str()));
+ ok.amount_index = num_elems;
+ }
+ else if (result != MDB_NOTFOUND)
+ throw0(DB_ERROR(lmdb_error("Failed to get output amount in db transaction: ", result).c_str()));
else
- {
- throw0(DB_ERROR("Wrong output type: expected txout_to_key"));
- }
+ ok.amount_index = 0;
+ ok.tx_index = m_num_outputs;
+ ok.data.pubkey = boost::get < txout_to_key > (tx_output.target).key;
+ ok.data.unlock_time = unlock_time;
+ ok.data.height = m_height;
+ data.mv_data = &ok;
+ data.mv_size = sizeof(ok);
+
+ if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP)))
+ throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str()));
+
+ amount_output_index = ok.amount_index;
+ global_output_index = m_num_outputs;
m_num_outputs++;
}
+// global_output_indices is now ignored
void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_index,
const std::vector<uint64_t>& amount_output_indices,
const std::vector<uint64_t>& global_output_indices)
@@ -783,17 +803,11 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const uint64_t tx_inde
int result = 0;
int num_outputs = amount_output_indices.size();
- std::unique_ptr<uint64_t[]> paired_indices(new uint64_t[2*num_outputs]);
- for (int i = 0; i < num_outputs; ++i)
- {
- paired_indices[2*i] = amount_output_indices[i];
- paired_indices[2*i+1] = global_output_indices[i];
- }
MDB_val_copy<uint64_t> k_tx_index(tx_index);
MDB_val v;
- v.mv_data = (void*)paired_indices.get();
- v.mv_size = sizeof(uint64_t) * 2 * num_outputs;
+ v.mv_data = (void *)amount_output_indices.data();
+ v.mv_size = sizeof(uint64_t) * num_outputs;
// LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size);
result = mdb_cursor_put(m_cur_tx_outputs, &k_tx_index, &v, MDB_APPEND);
@@ -805,22 +819,22 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_index, const transactio
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- // only need global_output_indices
+ // only need amount_output_indices
std::vector<uint64_t> amount_output_indices, global_output_indices;
get_amount_and_global_output_indices(tx_index, amount_output_indices, global_output_indices);
- if (global_output_indices.empty())
+ if (amount_output_indices.empty())
{
if (tx.vout.empty())
- LOG_PRINT_L2("tx has no outputs, so no global output indices");
+ LOG_PRINT_L2("tx has no outputs, so no output indices");
else
- throw0(DB_ERROR("tx has outputs, but no global output indices found"));
+ throw0(DB_ERROR("tx has outputs, but no output indices found"));
}
for (uint64_t i = tx.vout.size(); i > 0; --i)
{
const tx_out tx_output = tx.vout[i-1];
- remove_output(global_output_indices[i-1], tx_output.amount);
+ remove_output(amount_output_indices[i-1], tx_output.amount);
}
}
@@ -838,91 +852,41 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo
MDB_val_copy<uint64_t> k(out_index);
- auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL);
- if (result == MDB_NOTFOUND)
- {
- LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices");
- }
- else if (result)
- {
- throw1(DB_ERROR("Error adding removal of output tx index to db transaction"));
- }
-
- result = mdb_del(*m_write_txn, m_output_txs, &k, NULL);
- // if (result != 0 && result != MDB_NOTFOUND)
- // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction"));
+ auto result = mdb_del(*m_write_txn, m_output_txs, &k, NULL);
if (result == MDB_NOTFOUND)
{
LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs");
}
else if (result)
{
- throw1(DB_ERROR("Error adding removal of output tx hash to db transaction"));
- }
-
- result = mdb_del(*m_write_txn, m_output_keys, (MDB_val *)&zerokval, &k);
- if (result == MDB_NOTFOUND)
- {
- LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys");
+ throw1(DB_ERROR(lmdb_error("Error adding removal of output tx to db transaction", result).c_str()));
}
- else if (result)
- throw1(DB_ERROR("Error adding removal of output pubkey to db transaction"));
remove_amount_output_index(amount, out_index);
m_num_outputs--;
}
-void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index)
+void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t output_index)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
mdb_txn_cursors *m_cursors = &m_wcursors;
CURSOR(output_amounts);
- MDB_val_copy<uint64_t> k(amount);
- MDB_val v;
+ MDB_val_set(k, amount);
+ MDB_val_set(v, output_index);
- auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
+ auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
if (result == MDB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
- throw0(DB_ERROR("DB error attempting to get an output"));
-
- mdb_size_t num_elems = 0;
- mdb_cursor_count(m_cur_output_amounts, &num_elems);
-
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP);
+ throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str()));
- uint64_t amount_output_index = 0;
- uint64_t goi = 0;
- bool found_index = false;
- for (uint64_t i = num_elems; i > 0; --i)
- {
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT);
- goi = *(const uint64_t *)v.mv_data;
- if (goi == global_output_index)
- {
- amount_output_index = i-1;
- found_index = true;
- break;
- }
- if (i > 1)
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_DUP);
- }
- if (found_index)
- {
- // found the amount output index
- // now delete it
- result = mdb_cursor_del(m_cur_output_amounts, 0);
- if (result)
- throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str()));
- }
- else
- {
- // not found
- throw1(OUTPUT_DNE("Failed to find amount output index"));
- }
+ // now delete it
+ result = mdb_cursor_del(m_cur_output_amounts, 0);
+ if (result)
+ throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(output_index).append(": ")).c_str(), result).c_str()));
}
void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
@@ -950,7 +914,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image)
MDB_val k = {sizeof(k_image), (void *)&k_image};
auto result = mdb_del(*m_write_txn, m_spent_keys, (MDB_val *)&zerokval, &k);
if (result != 0 && result != MDB_NOTFOUND)
- throw1(DB_ERROR("Error adding removal of key image to db transaction"));
+ throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str()));
}
blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const
@@ -976,19 +940,6 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const
return o;
}
-uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index)
-{
- LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- std::vector <uint64_t> offsets;
- std::vector <uint64_t> global_indices;
- offsets.push_back(index);
- get_output_global_indices(amount, offsets, global_indices);
- if (!global_indices.size())
- throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
-
- return global_indices[0];
-}
-
void BlockchainLMDB::check_open() const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -1028,6 +979,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
{
+ int result;
+
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
if (m_open)
@@ -1057,10 +1010,10 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
m_folder = filename;
// set up lmdb environment
- if (mdb_env_create(&m_env))
- throw0(DB_ERROR("Failed to create lmdb environment"));
- if (mdb_env_set_maxdbs(m_env, 20))
- throw0(DB_ERROR("Failed to set max number of dbs"));
+ if ((result = mdb_env_create(&m_env)))
+ throw0(DB_ERROR(lmdb_error("Failed to create lmdb environment: ", result).c_str()));
+ if ((result = mdb_env_set_maxdbs(m_env, 20)))
+ throw0(DB_ERROR(lmdb_error("Failed to set max number of dbs: ", result).c_str()));
size_t mapsize = DEFAULT_MAPSIZE;
@@ -1107,9 +1060,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs");
lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs");
- lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices");
- lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_INTEGERDUP | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
- lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_output_keys, "Failed to open db handle for m_output_keys");
+ lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys");
@@ -1121,50 +1072,30 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
mdb_set_dupsort(txn, m_spent_keys, compare_hash32);
mdb_set_dupsort(txn, m_block_heights, compare_hash32);
mdb_set_dupsort(txn, m_tx_indices, compare_hash32);
- mdb_set_dupsort(txn, m_output_keys, compare_uint64);
+ mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
mdb_set_compare(txn, m_hf_starting_heights, compare_uint8);
mdb_set_compare(txn, m_properties, compare_string);
// get and keep current height
MDB_stat db_stats;
- if (mdb_stat(txn, m_blocks, &db_stats))
- throw0(DB_ERROR("Failed to query m_blocks"));
+ if ((result = mdb_stat(txn, m_blocks, &db_stats)))
+ throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries);
m_height = db_stats.ms_entries;
// get and keep current number of txs
- if (mdb_stat(txn, m_tx_indices, &db_stats))
- throw0(DB_ERROR("Failed to query m_tx_indices"));
+ if ((result = mdb_stat(txn, m_tx_indices, &db_stats)))
+ throw0(DB_ERROR(lmdb_error("Failed to query m_tx_indices: ", result).c_str()));
m_num_txs = db_stats.ms_entries;
// get and keep current number of outputs
- if (mdb_stat(txn, m_output_indices, &db_stats))
- throw0(DB_ERROR("Failed to query m_output_indices"));
+ if ((result = mdb_stat(txn, m_output_txs, &db_stats)))
+ throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
m_num_outputs = db_stats.ms_entries;
bool compatible = true;
- // ND: This "new" version of the lmdb database is incompatible with
- // the previous version. Ensure that the output_keys database is
- // sizeof(output_data_t) in length. Otherwise, inform user and
- // terminate.
- if(m_height > 0)
- {
- MDB_val v;
- auto get_result = mdb_get(txn, m_output_keys, (MDB_val *)&zerokval, &v);
- if(get_result != MDB_SUCCESS)
- {
- txn.abort();
- m_open = false;
- return;
- }
-
- // LOG_PRINT_L0("Output keys size: " << v.mv_size);
- if(v.mv_size != sizeof(outkey))
- compatible = false;
- }
-
MDB_val_copy<const char*> k("version");
MDB_val v;
auto get_result = mdb_get(txn, m_properties, &k, &v);
@@ -1260,17 +1191,15 @@ void BlockchainLMDB::reset()
check_open();
mdb_txn_safe txn;
- if (mdb_txn_begin(m_env, NULL, 0, txn))
- throw0(DB_ERROR("Failed to create a transaction for the db"));
+ if (auto result = mdb_txn_begin(m_env, NULL, 0, txn))
+ throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
mdb_drop(txn, m_blocks, 0);
mdb_drop(txn, m_block_info, 0);
mdb_drop(txn, m_block_heights, 0);
mdb_drop(txn, m_txs, 0);
mdb_drop(txn, m_tx_outputs, 0);
mdb_drop(txn, m_output_txs, 0);
- mdb_drop(txn, m_output_indices, 0);
mdb_drop(txn, m_output_amounts, 0);
- mdb_drop(txn, m_output_keys, 0);
mdb_drop(txn, m_spent_keys, 0);
mdb_drop(txn, m_hf_starting_heights, 0);
mdb_drop(txn, m_hf_versions, 0);
@@ -1332,10 +1261,12 @@ void BlockchainLMDB::unlock()
} \
#define TXN_PREFIX_RDONLY() \
- bool my_rtxn = block_rtxn_start(); \
- MDB_txn *m_txn = m_write_txn ? m_write_txn->m_txn : m_tinfo->m_ti_rtxn
-#define TXN_POSTFIX_RDONLY() \
- if (my_rtxn) block_rtxn_stop()
+ MDB_txn *m_txn; \
+ mdb_txn_cursors *m_cursors; \
+ bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \
+ mdb_txn_safe auto_txn(my_rtxn); \
+ if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get()
+#define TXN_POSTFIX_RDONLY()
#define TXN_POSTFIX_SUCCESS() \
do { \
@@ -1375,23 +1306,23 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(block_heights);
+ bool ret = false;
blk_height bh = {h, 0};
MDB_val key = {sizeof(bh), (void *)&bh};
auto get_result = mdb_cursor_get(m_cur_block_heights, (MDB_val *)&zerokval, &key, MDB_GET_BOTH);
if (get_result == MDB_NOTFOUND)
{
- TXN_POSTFIX_RDONLY();
LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
- return false;
}
else if (get_result)
- throw0(DB_ERROR("DB error attempting to fetch block index from hash"));
+ throw0(DB_ERROR(lmdb_error("DB error attempting to fetch block index from hash", get_result).c_str()));
+ else
+ ret = true;
TXN_POSTFIX_RDONLY();
- return true;
+ return ret;
}
block BlockchainLMDB::get_block(const crypto::hash& h) const
@@ -1408,7 +1339,6 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(block_heights);
blk_height bh = {h, 0};
@@ -1440,7 +1370,6 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(blocks);
MDB_val_copy<uint64_t> key(height);
@@ -1471,7 +1400,6 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(block_info);
MDB_val_copy<uint64_t> key(height);
@@ -1510,7 +1438,6 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(block_info);
MDB_val_copy<uint64_t> key(height);
@@ -1535,7 +1462,6 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t&
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(block_info);
MDB_val_copy<uint64_t> key(height);
@@ -1577,7 +1503,6 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(block_info);
MDB_val_copy<uint64_t> key(height);
@@ -1602,7 +1527,6 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height)
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(block_info);
MDB_val_copy<uint64_t> key(height);
@@ -1689,7 +1613,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
RCURSOR(txs);
@@ -1730,7 +1653,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
txindex ti = {h};
@@ -1747,14 +1669,17 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_index) const
TXN_POSTFIX_RDONLY();
+ bool ret = false;
if (get_result == MDB_NOTFOUND)
{
LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
- return false;
}
else if (get_result)
- throw0(DB_ERROR("DB error attempting to fetch transaction from hash"));
- return true;
+ throw0(DB_ERROR(lmdb_error("DB error attempting to fetch transaction from hash", get_result).c_str()));
+ else
+ ret = true;
+
+ return ret;
}
uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const
@@ -1763,7 +1688,6 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
txindex ti = {h};
@@ -1786,7 +1710,6 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
RCURSOR(txs);
@@ -1804,7 +1727,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
if (get_result == MDB_NOTFOUND)
throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
else if (get_result)
- throw0(DB_ERROR("DB error attempting to fetch tx from hash"));
+ throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
blobdata bd;
bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size);
@@ -1854,7 +1777,6 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
txindex ti = {h};
@@ -1865,7 +1787,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
throw1(TX_DNE(std::string("tx_data_t with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
}
else if (get_result)
- throw0(DB_ERROR("DB error attempting to fetch tx height from hash"));
+ throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx height from hash", get_result).c_str()));
txindex *tip = (txindex *)v.mv_data;
uint64_t ret = tip->data.height;
@@ -1879,42 +1801,45 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(output_amounts);
MDB_val_copy<uint64_t> k(amount);
MDB_val v;
+ mdb_size_t num_elems = 0;
auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
- if (result == MDB_NOTFOUND)
+ if (result == MDB_SUCCESS)
{
- TXN_POSTFIX_RDONLY();
- return 0;
+ mdb_cursor_count(m_cur_output_amounts, &num_elems);
}
- else if (result)
+ else if (result != MDB_NOTFOUND)
throw0(DB_ERROR("DB error attempting to get number of outputs of an amount"));
- mdb_size_t num_elems = 0;
- mdb_cursor_count(m_cur_output_amounts, &num_elems);
-
TXN_POSTFIX_RDONLY();
return num_elems;
}
+// TODO: probably remove this function
output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const
{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)");
+ outkey ok = {0};
+ return ok.data;
+}
+
+output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index)
+{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
- RCURSOR(output_keys);
+ RCURSOR(output_amounts);
- outkey ok = {global_index};
- MDB_val v = {sizeof(ok), (void *)&ok};
- auto get_result = mdb_cursor_get(m_cur_output_keys, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+ MDB_val_set(k, amount);
+ MDB_val_set(v, index);
+ auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
if (get_result == MDB_NOTFOUND)
- throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist"));
+ throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist"));
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
outkey *okp = (outkey *)v.mv_data;
@@ -1923,24 +1848,13 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const
return ret;
}
-output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index)
-{
- LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- check_open();
-
- uint64_t glob_index = get_output_global_index(amount, index);
- return get_output_key(glob_index);
-}
-
tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(output_txs);
- RCURSOR(output_indices);
MDB_val_copy<uint64_t> k(index);
MDB_val v;
@@ -1951,15 +1865,9 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
- crypto::hash tx_hash = *(const crypto::hash*)v.mv_data;
-
- get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET);
- if (get_result == MDB_NOTFOUND)
- throw1(OUTPUT_DNE("output with given index not in db"));
- else if (get_result)
- throw0(DB_ERROR("DB error attempting to fetch output tx index"));
+ outtx *ot = (outtx *)v.mv_data;
+ tx_out_index ret = tx_out_index(ot->tx_hash, ot->local_index);
- tx_out_index ret = tx_out_index(tx_hash, *(const uint64_t *)v.mv_data);
TXN_POSTFIX_RDONLY();
return ret;
}
@@ -1977,6 +1885,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
return indices[0];
}
+// we don't have global_output_indices
void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_index,
std::vector<uint64_t>& amount_output_indices,
std::vector<uint64_t>& global_output_indices) const
@@ -1992,7 +1901,6 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_inde
// txn, regardless of batch mode. Otherwise, remove_tx_outputs() would now
// create a new read-only txn here, which is incorrect.
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_outputs);
int result = 0;
@@ -2001,25 +1909,21 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const uint64_t tx_inde
result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_index, &v, MDB_SET);
if (result == MDB_NOTFOUND)
- LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in "
+ LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in "
"tx_outputs, but it should have an empty entry even if it's a tx without "
"outputs");
else if (result)
- throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_index]"));
+ throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str()));
- uint64_t* paired_indices = (uint64_t*)v.mv_data;
- int num_elems = v.mv_size / sizeof(uint64_t);
- if (num_elems % 2 != 0)
- throw0(DB_ERROR("tx_outputs[tx_index] does not have an even numer of indices"));
- int num_outputs = num_elems / 2;
+ uint64_t* indices = (uint64_t*)v.mv_data;
+ int num_outputs = v.mv_size / sizeof(uint64_t);
for (int i = 0; i < num_outputs; ++i)
{
// LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]);
- amount_output_indices.push_back(paired_indices[2*i]);
- global_output_indices.push_back(paired_indices[2*i+1]);
+ amount_output_indices.push_back(indices[i]);
}
- paired_indices = nullptr;
+ indices = nullptr;
TXN_POSTFIX_RDONLY();
}
@@ -2045,11 +1949,11 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
bool ret;
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(spent_keys);
MDB_val k = {sizeof(img), (void *)&img};
ret = (mdb_cursor_get(m_cur_spent_keys, (MDB_val *)&zerokval, &k, MDB_GET_BOTH) == 0);
+
TXN_POSTFIX_RDONLY();
return ret;
}
@@ -2060,7 +1964,6 @@ bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_ima
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(spent_keys);
MDB_val k;
@@ -2093,7 +1996,6 @@ bool BlockchainLMDB::for_all_blocks(std::function<bool(uint64_t, const crypto::h
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(blocks);
MDB_val k;
@@ -2135,7 +2037,6 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(txs);
MDB_val k;
@@ -2174,7 +2075,6 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(output_amounts);
MDB_val k;
@@ -2191,8 +2091,8 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c
if (ret)
throw0(DB_ERROR("Failed to enumerate outputs"));
uint64_t amount = *(const uint64_t*)k.mv_data;
- uint64_t global_index = *(const uint64_t*)v.mv_data;
- tx_out_index toi = get_output_tx_and_index_from_global(global_index);
+ outkey *ok = (outkey *)v.mv_data;
+ tx_out_index toi = get_output_tx_and_index_from_global(ok->tx_index);
if (!f(amount, toi.first, toi.second)) {
ret = false;
break;
@@ -2218,6 +2118,7 @@ void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
check_open();
+ m_writer = boost::this_thread::get_id();
check_and_resize_for_batch(batch_num_blocks);
m_write_batch_txn = new mdb_txn_safe();
@@ -2260,6 +2161,7 @@ void BlockchainLMDB::batch_commit()
m_write_txn = nullptr;
delete m_write_batch_txn;
+ m_write_batch_txn = nullptr;
memset(&m_wcursors, 0, sizeof(m_wcursors));
}
@@ -2299,8 +2201,9 @@ void BlockchainLMDB::batch_abort()
m_write_txn = nullptr;
// explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
m_write_batch_txn->abort();
- m_batch_active = false;
+ delete m_write_batch_txn;
m_write_batch_txn = nullptr;
+ m_batch_active = false;
memset(&m_wcursors, 0, sizeof(m_wcursors));
LOG_PRINT_L3("batch transaction: aborted");
}
@@ -2317,10 +2220,14 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions)
}
// return true if we started the txn, false if already started
-bool BlockchainLMDB::block_rtxn_start() const
+bool BlockchainLMDB::block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) const
{
- if (m_write_txn)
- return false;
+ bool ret = false;
+ if (m_write_txn && m_writer == boost::this_thread::get_id()) {
+ *mtxn = m_write_txn->m_txn;
+ *mcur = (mdb_txn_cursors *)&m_wcursors;
+ return ret;
+ }
if (!m_tinfo.get())
{
m_tinfo.reset(new mdb_threadinfo);
@@ -2328,17 +2235,21 @@ bool BlockchainLMDB::block_rtxn_start() const
memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str()));
+ ret = true;
} else if (!m_tinfo->m_ti_rflags.m_rf_txn)
{
if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn))
throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str()));
- } else
- {
- return false;
+ ret = true;
}
- m_tinfo->m_ti_rflags.m_rf_txn = true;
- LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- return true;
+ if (ret)
+ m_tinfo->m_ti_rflags.m_rf_txn = true;
+ *mtxn = m_tinfo->m_ti_rtxn;
+ *mcur = &m_tinfo->m_ti_rcursors;
+
+ if (ret)
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ return ret;
}
void BlockchainLMDB::block_rtxn_stop() const
@@ -2353,7 +2264,7 @@ void BlockchainLMDB::block_txn_start(bool readonly)
if (readonly)
{
bool didit = false;
- if (m_write_txn)
+ if (m_write_txn && m_writer == boost::this_thread::get_id())
return;
if (!m_tinfo.get())
{
@@ -2389,6 +2300,7 @@ void BlockchainLMDB::block_txn_start(bool readonly)
throw0(DB_ERROR_TXN_START((std::string("Attempted to start new write txn when write txn already exists in ")+__FUNCTION__).c_str()));
if (! m_batch_active)
{
+ m_writer = boost::this_thread::get_id();
m_write_txn = new mdb_txn_safe();
if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_txn))
{
@@ -2403,9 +2315,9 @@ void BlockchainLMDB::block_txn_start(bool readonly)
void BlockchainLMDB::block_txn_stop()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- if (! m_batch_active)
+ if (m_write_txn && m_writer == boost::this_thread::get_id())
{
- if (m_write_txn)
+ if (! m_batch_active)
{
TIME_MEASURE_START(time1);
m_write_txn->commit();
@@ -2416,38 +2328,38 @@ void BlockchainLMDB::block_txn_stop()
m_write_txn = nullptr;
memset(&m_wcursors, 0, sizeof(m_wcursors));
}
- else if (m_tinfo->m_ti_rtxn)
- {
- mdb_txn_reset(m_tinfo->m_ti_rtxn);
- memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
- }
+ }
+ else if (m_tinfo->m_ti_rtxn)
+ {
+ mdb_txn_reset(m_tinfo->m_ti_rtxn);
+ memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
}
}
void BlockchainLMDB::block_txn_abort()
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- if (! m_batch_active)
+ if (m_write_txn && m_writer == boost::this_thread::get_id())
{
- if (m_write_txn != nullptr)
+ if (! m_batch_active)
{
delete m_write_txn;
m_write_txn = nullptr;
memset(&m_wcursors, 0, sizeof(m_wcursors));
}
- else if (m_tinfo->m_ti_rtxn)
- {
- mdb_txn_reset(m_tinfo->m_ti_rtxn);
- memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
- }
- else
- {
- // This would probably mean an earlier exception was caught, but then we
- // proceeded further than we should have.
- throw0(DB_ERROR((std::string("BlockchainLMDB::") + __func__ +
- std::string(": block-level DB transaction abort called when write txn doesn't exist")
- ).c_str()));
- }
+ }
+ else if (m_tinfo->m_ti_rtxn)
+ {
+ mdb_txn_reset(m_tinfo->m_ti_rtxn);
+ memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
+ }
+ else
+ {
+ // This would probably mean an earlier exception was caught, but then we
+ // proceeded further than we should have.
+ throw0(DB_ERROR((std::string("BlockchainLMDB::") + __func__ +
+ std::string(": block-level DB transaction abort called when write txn doesn't exist")
+ ).c_str()));
}
}
@@ -2521,9 +2433,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6
tx_out_indices.clear();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(output_txs);
- RCURSOR(output_indices);
for (const uint64_t &index : global_indices)
{
@@ -2536,201 +2446,77 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
- crypto::hash tx_hash = *(const crypto::hash*) v.mv_data;
-
- get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET);
- if (get_result == MDB_NOTFOUND)
- throw1(OUTPUT_DNE("output with given index not in db"));
- else if (get_result)
- throw0(DB_ERROR("DB error attempting to fetch output tx index"));
-
- auto result = tx_out_index(tx_hash, *(const uint64_t *) v.mv_data);
+ outtx *ot = (outtx *)v.mv_data;
+ auto result = tx_out_index(ot->tx_hash, ot->local_index);
tx_out_indices.push_back(result);
}
TXN_POSTFIX_RDONLY();
}
-void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets,
- std::vector<uint64_t> &global_indices)
+void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- TIME_MEASURE_START(txx);
+ TIME_MEASURE_START(db3);
check_open();
- global_indices.clear();
-
- uint64_t max = 0;
- for (const uint64_t &index : offsets)
- {
- if (index > max)
- max = index;
- }
+ outputs.clear();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
- RCURSOR(output_amounts);
-
- MDB_val_copy<uint64_t> k(amount);
- MDB_val v;
- auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
- if (result == MDB_NOTFOUND)
- throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
- else if (result)
- throw0(DB_ERROR("DB error attempting to get an output"));
- mdb_size_t num_elems = 0;
- mdb_cursor_count(m_cur_output_amounts, &num_elems);
- if (max <= 1 && num_elems <= max)
- throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
+ RCURSOR(output_amounts);
- uint64_t t_dbmul = 0;
- uint64_t t_dbscan = 0;
- if (max <= 1)
+ MDB_val_set(k, amount);
+ for (const uint64_t &index : offsets)
{
- for (const uint64_t& index : offsets)
- {
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP);
- for (uint64_t i = 0; i < index; ++i)
- {
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP);
- }
+ MDB_val_set(v, index);
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT);
- uint64_t glob_index = *(const uint64_t*) v.mv_data;
- LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index);
- global_indices.push_back(glob_index);
- }
- }
- else
- {
- uint32_t curcount = 0;
- uint32_t blockstart = 0;
- for (const uint64_t& index : offsets)
- {
- if (index >= num_elems)
- {
- LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index");
- break;
- }
- if (!curcount && index > num_elems/2)
- {
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP);
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV); /* kludge to unset C_EOF */
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT);
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_MULTIPLE);
-
- curcount = num_elems;
- while(1)
- {
- TIME_MEASURE_START(db1);
- int count = v.mv_size / sizeof(uint64_t);
- curcount -= count;
- if (curcount > index)
- {
- mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_MULTIPLE);
- } else
- {
- blockstart = curcount;
- curcount += count;
- break;
- }
- TIME_MEASURE_FINISH(db1);
- t_dbmul += db1;
- }
-
- } else
- {
- while (index >= curcount)
- {
- TIME_MEASURE_START(db1);
- if (mdb_cursor_get(m_cur_output_amounts, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0)
- {
- // allow partial results
- result = false;
- break;
- }
-
- int count = v.mv_size / sizeof(uint64_t);
-
- blockstart = curcount;
- curcount += count;
- TIME_MEASURE_FINISH(db1);
- t_dbmul += db1;
- }
- }
-
- LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index);
- TIME_MEASURE_START(db2);
- uint64_t actual_index = index - blockstart;
- uint64_t glob_index = ((const uint64_t*) v.mv_data)[actual_index];
-
- LOG_PRINT_L3("Amount: " << amount << " M1->v: " << glob_index);
- global_indices.push_back(glob_index);
-
- TIME_MEASURE_FINISH(db2);
- t_dbscan += db2;
+ auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
+ if (get_result == MDB_NOTFOUND)
+ throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist"));
+ else if (get_result)
+ throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str()));
- }
+ outkey *okp = (outkey *)v.mv_data;
+ output_data_t data = okp->data;
+ outputs.push_back(data);
}
TXN_POSTFIX_RDONLY();
- TIME_MEASURE_FINISH(txx);
- LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan);
+ TIME_MEASURE_FINISH(db3);
+ LOG_PRINT_L3("db3: " << db3);
}
-void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs)
+void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- TIME_MEASURE_START(db3);
check_open();
- outputs.clear();
+ indices.clear();
+ std::vector <uint64_t> tx_indices;
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
- std::vector <uint64_t> global_indices;
- get_output_global_indices(amount, offsets, global_indices);
- if (global_indices.size() > 0)
+ RCURSOR(output_amounts);
+
+ MDB_val_set(k, amount);
+ for (const uint64_t &index : offsets)
{
- RCURSOR(output_keys);
+ MDB_val_set(v, index);
- for (const uint64_t &index : global_indices)
- {
- outkey ok = {index};
- MDB_val v = {sizeof(ok), (void *)&ok};
-
- auto get_result = mdb_cursor_get(m_cur_output_keys, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
- if (get_result == MDB_NOTFOUND)
- throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist"));
- else if (get_result)
- throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
-
- outkey *okp = (outkey *)v.mv_data;
- output_data_t data = okp->data;
- outputs.push_back(data);
- }
+ auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
+ if (get_result == MDB_NOTFOUND)
+ throw1(OUTPUT_DNE("Attempting to get output by index, but key does not exist"));
+ else if (get_result)
+ throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output from the db", get_result).c_str()));
+ outkey *okp = (outkey *)v.mv_data;
+ tx_indices.push_back(okp->tx_index);
}
- TXN_POSTFIX_RDONLY();
-
- TIME_MEASURE_FINISH(db3);
- LOG_PRINT_L3("db3: " << db3);
-}
-
-void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices)
-{
- LOG_PRINT_L3("BlockchainLMDB::" << __func__);
- check_open();
- indices.clear();
-
- std::vector <uint64_t> global_indices;
- get_output_global_indices(amount, offsets, global_indices);
TIME_MEASURE_START(db3);
- if(global_indices.size() > 0)
+ if(tx_indices.size() > 0)
{
- get_output_tx_and_index_from_global(global_indices, indices);
+ get_output_tx_and_index_from_global(tx_indices, indices);
}
TIME_MEASURE_FINISH(db3);
LOG_PRINT_L3("db3: " << db3);
@@ -2800,18 +2586,21 @@ uint64_t BlockchainLMDB::get_hard_fork_starting_height(uint8_t version) const
MDB_val_copy<uint8_t> val_key(version);
MDB_val val_ret;
+ uint64_t ret = 0;
auto result = mdb_get(m_txn, m_hf_starting_heights, &val_key, &val_ret);
- if (result == MDB_NOTFOUND)
- return std::numeric_limits<uint64_t>::max();
- if (result)
- throw0(DB_ERROR("Error attempting to retrieve a hard fork starting height from the db"));
-
- uint64_t ret;
+ if (result == MDB_SUCCESS)
+ {
#ifdef MISALIGNED_OK
- ret = *(const uint64_t*)val_ret.mv_data;
+ ret = *(const uint64_t*)val_ret.mv_data;
#else
- memcpy(&ret, val_ret.mv_data, sizeof(uint64_t));
+ memcpy(&ret, val_ret.mv_data, sizeof(uint64_t));
#endif
+ } else if (result == MDB_NOTFOUND)
+ {
+ ret = std::numeric_limits<uint64_t>::max();
+ } else if (result)
+ throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a hard fork starting height from the db", result).c_str()));
+
TXN_POSTFIX_RDONLY();
return ret;
}
@@ -2841,7 +2630,6 @@ uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const
check_open();
TXN_PREFIX_RDONLY();
- const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(hf_versions);
MDB_val_copy<uint64_t> val_key(height);
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index 0eda3bc4d..a298ee68f 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -46,9 +46,7 @@ typedef struct mdb_txn_cursors
MDB_cursor *m_txc_block_info;
MDB_cursor *m_txc_output_txs;
- MDB_cursor *m_txc_output_indices;
MDB_cursor *m_txc_output_amounts;
- MDB_cursor *m_txc_output_keys;
MDB_cursor *m_txc_txs;
MDB_cursor *m_txc_tx_indices;
@@ -63,9 +61,7 @@ typedef struct mdb_txn_cursors
#define m_cur_block_heights m_cursors->m_txc_block_heights
#define m_cur_block_info m_cursors->m_txc_block_info
#define m_cur_output_txs m_cursors->m_txc_output_txs
-#define m_cur_output_indices m_cursors->m_txc_output_indices
#define m_cur_output_amounts m_cursors->m_txc_output_amounts
-#define m_cur_output_keys m_cursors->m_txc_output_keys
#define m_cur_txs m_cursors->m_txc_txs
#define m_cur_tx_indices m_cursors->m_txc_tx_indices
#define m_cur_tx_outputs m_cursors->m_txc_tx_outputs
@@ -79,9 +75,7 @@ typedef struct mdb_rflags
bool m_rf_block_heights;
bool m_rf_block_info;
bool m_rf_output_txs;
- bool m_rf_output_indices;
bool m_rf_output_amounts;
- bool m_rf_output_keys;
bool m_rf_txs;
bool m_rf_tx_indices;
bool m_rf_tx_outputs;
@@ -100,7 +94,7 @@ typedef struct mdb_threadinfo
struct mdb_txn_safe
{
- mdb_txn_safe();
+ mdb_txn_safe(const bool check=true);
~mdb_txn_safe();
void commit(std::string message = "");
@@ -127,8 +121,10 @@ struct mdb_txn_safe
static void wait_no_active_txns();
static void allow_new_txns();
+ mdb_threadinfo* m_tinfo;
MDB_txn* m_txn;
bool m_batch_txn = false;
+ bool m_check;
static std::atomic<uint64_t> num_active_txns;
// could use a mutex here, but this should be sufficient.
@@ -230,7 +226,6 @@ public:
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
- virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices);
virtual void get_amount_and_global_output_indices(const uint64_t tx_index,
std::vector<uint64_t>& amount_output_indices,
@@ -261,7 +256,7 @@ public:
virtual void block_txn_start(bool readonly);
virtual void block_txn_stop();
virtual void block_txn_abort();
- virtual bool block_rtxn_start() const;
+ virtual bool block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) const;
virtual void block_rtxn_stop() const;
virtual void pop_block(block& blk, std::vector<transaction>& txs);
@@ -337,16 +332,6 @@ private:
*/
tx_out output_from_blob(const blobdata& blob) const;
- /**
- * @brief get the global index of the index-th output of the given amount
- *
- * @param amount the output amount
- * @param index the index into the set of outputs of that amount
- *
- * @return the global index of the desired output
- */
- uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
-
void check_open() const;
virtual bool is_read_only() const;
@@ -365,9 +350,7 @@ private:
MDB_dbi m_tx_outputs;
MDB_dbi m_output_txs;
- MDB_dbi m_output_indices;
MDB_dbi m_output_amounts;
- MDB_dbi m_output_keys;
MDB_dbi m_spent_keys;
@@ -384,6 +367,7 @@ private:
std::string m_folder;
mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
+ boost::thread::id m_writer;
bool m_batch_transactions; // support for batch transactions
bool m_batch_active; // whether batch transaction is in progress
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-20 15:15:30 +0000