aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--contrib/epee/include/net/abstract_tcp_server2.inl2
-rw-r--r--external/db_drivers/liblmdb/lmdb.h19
-rw-r--r--external/db_drivers/liblmdb/mdb.c80
-rw-r--r--external/db_drivers/liblmdb/midl.h2
-rw-r--r--src/blockchain_db/berkeleydb/db_bdb.cpp4
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.cpp182
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.h44
-rw-r--r--src/blockchain_utilities/CMakeLists.txt10
-rw-r--r--src/blockchain_utilities/blockchain_converter.cpp4
-rw-r--r--src/blockchain_utilities/blockchain_import.cpp190
-rw-r--r--src/blockchain_utilities/fake_core.h26
-rw-r--r--src/wallet/wallet2.cpp4
-rw-r--r--tests/core_tests/chaingen.h2
13 files changed, 441 insertions, 128 deletions
diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl
index 934132ea2..4fb6fa75d 100644
--- a/contrib/epee/include/net/abstract_tcp_server2.inl
+++ b/contrib/epee/include/net/abstract_tcp_server2.inl
@@ -144,7 +144,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
// create a random uuid
boost::uuids::uuid random_uuid;
// that stuff turns out to be included, even though it's from src... Taking advantage
- crypto::generate_random_bytes(sizeof(random_uuid), &random_uuid);
+ random_uuid = crypto::rand<boost::uuids::uuid>();
context.set_details(random_uuid, ip_, remote_ep.port(), is_income);
_dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
diff --git a/external/db_drivers/liblmdb/lmdb.h b/external/db_drivers/liblmdb/lmdb.h
index d616cabf9..a794c9f7d 100644
--- a/external/db_drivers/liblmdb/lmdb.h
+++ b/external/db_drivers/liblmdb/lmdb.h
@@ -78,6 +78,11 @@
* access to locks and lock file. Exceptions: On read-only filesystems
* or with the #MDB_NOLOCK flag described under #mdb_env_open().
*
+ * - An LMDB configuration will often reserve considerable \b unused
+ * memory address space and maybe file size for future growth.
+ * This does not use actual memory or disk space, but users may need
+ * to understand the difference so they won't be scared off.
+ *
* - By default, in versions before 0.9.10, unused portions of the data
* file might receive garbage data from memory freed by other code.
* (This does not happen when using the #MDB_WRITEMAP flag.) As of
@@ -130,7 +135,7 @@
*
* @author Howard Chu, Symas Corporation.
*
- * @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
+ * @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
@@ -392,7 +397,9 @@ typedef enum MDB_cursor_op {
MDB_PREV_NODUP, /**< Position at last data item of previous key */
MDB_SET, /**< Position at specified key */
MDB_SET_KEY, /**< Position at specified key, return key + data */
- MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */
+ MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */
+ MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to
+ a page of duplicate data items. Only for #MDB_DUPFIXED */
} MDB_cursor_op;
/** @defgroup errors Return Codes
@@ -538,9 +545,11 @@ int mdb_env_create(MDB_env **env);
* allowed. LMDB will still modify the lock file - except on read-only
* filesystems, where LMDB does not use locks.
* <li>#MDB_WRITEMAP
- * Use a writeable memory map unless MDB_RDONLY is set. This is faster
- * and uses fewer mallocs, but loses protection from application bugs
+ * Use a writeable memory map unless MDB_RDONLY is set. This uses
+ * fewer mallocs but loses protection from application bugs
* like wild pointer writes and other bad updates into the database.
+ * This may be slightly faster for DBs that fit entirely in RAM, but
+ * is slower for DBs larger than RAM.
* Incompatible with nested transactions.
* Do not mix processes with and without MDB_WRITEMAP on the same
* environment. This can defeat durability (#mdb_env_sync etc).
@@ -1535,7 +1544,7 @@ int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags);
* <li>EINVAL - cursor is not initialized, or an invalid parameter was specified.
* </ul>
*/
-int mdb_cursor_count(MDB_cursor *cursor, size_t *countp);
+int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp);
/** @brief Compare two data items according to a particular database.
*
diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c
index 7be13997a..a366b4024 100644
--- a/external/db_drivers/liblmdb/mdb.c
+++ b/external/db_drivers/liblmdb/mdb.c
@@ -5,7 +5,7 @@
* BerkeleyDB API, but much simplified.
*/
/*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -1075,7 +1075,7 @@ typedef struct MDB_db {
pgno_t md_branch_pages; /**< number of internal pages */
pgno_t md_leaf_pages; /**< number of leaf pages */
pgno_t md_overflow_pages; /**< number of overflow pages */
- pgno_t md_entries; /**< number of data items */
+ mdb_size_t md_entries; /**< number of data items */
pgno_t md_root; /**< the root page of this tree */
} MDB_db;
@@ -1426,7 +1426,7 @@ typedef struct MDB_ntxn {
#endif
/** max bytes to write in one call */
-#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4))
+#define MAX_WRITE (0x40000000U >> (sizeof(ssize_t) == 4))
/** Check \b txn and \b dbi arguments to a function */
#define TXN_DBI_EXIST(txn, dbi, validity) \
@@ -4160,6 +4160,19 @@ mdb_env_create(MDB_env **env)
return MDB_SUCCESS;
}
+#ifdef _WIN32
+/** @brief Map a result from an NTAPI call to WIN32. */
+static DWORD
+mdb_nt2win32(NTSTATUS st)
+{
+ OVERLAPPED o = {0};
+ DWORD br;
+ o.Internal = st;
+ GetOverlappedResult(NULL, &o, &br, FALSE);
+ return GetLastError();
+}
+#endif
+
static int ESECT
mdb_env_map(MDB_env *env, void *addr)
{
@@ -4189,7 +4202,7 @@ mdb_env_map(MDB_env *env, void *addr)
rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd);
if (rc)
- return rc;
+ return mdb_nt2win32(rc);
map = addr;
#ifdef MDB_VL32
msize = NUM_METAS * env->me_psize;
@@ -4201,7 +4214,7 @@ mdb_env_map(MDB_env *env, void *addr)
NtClose(mh);
#endif
if (rc)
- return rc;
+ return mdb_nt2win32(rc);
env->me_map = map;
#else
#ifdef MDB_VL32
@@ -5256,14 +5269,16 @@ mdb_env_close0(MDB_env *env, int excl)
free(env->me_dbflags);
free(env->me_path);
free(env->me_dirty_list);
- free(env->me_txn0);
#ifdef MDB_VL32
+ if (env->me_txn0 && env->me_txn0->mt_rpages)
+ free(env->me_txn0->mt_rpages);
{ unsigned int x;
for (x=1; x<=env->me_rpages[0].mid; x++)
munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize);
}
free(env->me_rpages);
#endif
+ free(env->me_txn0);
mdb_midl_free(env->me_free_pgs);
if (env->me_flags & MDB_ENV_TXKEY) {
@@ -5691,7 +5706,8 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
#define MAP(rc,env,addr,len,off) \
addr = NULL; \
rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \
- len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY)
+ len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \
+ if (rc) rc = mdb_nt2win32(rc)
#else
off_t off;
size_t len;
@@ -5765,7 +5781,7 @@ notlocal:
pthread_mutex_lock(&env->me_rpmutex);
retry:
y = 0;
- for (i=1; i<tl[0].mid; i++) {
+ for (i=1; i<=tl[0].mid; i++) {
if (!tl[i].mref) {
if (!y) y = i;
/* tmp overflow pages don't go to env */
@@ -5845,7 +5861,7 @@ retry:
if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) {
/* purge unref'd pages */
unsigned i, y = 0;
- for (i=1; i<el[0].mid; i++) {
+ for (i=1; i<=el[0].mid; i++) {
if (!el[i].mref) {
if (!y) y = i;
munmap(el[i].mptr, env->me_psize * el[i].mcnt);
@@ -6436,8 +6452,10 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
DPRINTF(("cursor_next: top page is %"Y"u in cursor %p",
mdb_dbg_pgno(mp), (void *) mc));
- if (mc->mc_flags & C_DEL)
+ if (mc->mc_flags & C_DEL) {
+ mc->mc_flags ^= C_DEL;
goto skip;
+ }
if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) {
DPUTS("=====> move to next sibling page");
@@ -6523,6 +6541,8 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p",
mdb_dbg_pgno(mp), (void *) mc));
+ mc->mc_flags &= ~(C_EOF|C_DEL);
+
if (mc->mc_ki[mc->mc_top] == 0) {
DPUTS("=====> move to prev sibling page");
if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) {
@@ -6977,6 +6997,30 @@ fetchm:
}
}
break;
+ case MDB_PREV_MULTIPLE:
+ if (data == NULL) {
+ rc = EINVAL;
+ break;
+ }
+ if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
+ rc = MDB_INCOMPATIBLE;
+ break;
+ }
+ if (!(mc->mc_flags & C_INITIALIZED))
+ rc = mdb_cursor_last(mc, key, data);
+ else
+ rc = MDB_SUCCESS;
+ if (rc == MDB_SUCCESS) {
+ MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
+ if (mx->mc_flags & C_INITIALIZED) {
+ rc = mdb_cursor_sibling(mx, 0);
+ if (rc == MDB_SUCCESS)
+ goto fetchm;
+ } else {
+ rc = MDB_NOTFOUND;
+ }
+ }
+ break;
case MDB_NEXT:
case MDB_NEXT_DUP:
case MDB_NEXT_NODUP:
@@ -7518,7 +7562,7 @@ new_sub:
*/
if (do_sub) {
int xflags, new_dupdata;
- size_t ecount;
+ mdb_size_t ecount;
put_sub:
xdata.mv_size = 0;
xdata.mv_data = "";
@@ -8233,7 +8277,7 @@ mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc)
/* Return the count of duplicate data items for the current key */
int
-mdb_cursor_count(MDB_cursor *mc, size_t *countp)
+mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp)
{
MDB_node *leaf;
@@ -9957,7 +10001,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
MDB_txn *txn = NULL;
mdb_mutexref_t wmutex = NULL;
int rc;
- size_t wsize;
+ mdb_size_t wsize, w3;
char *ptr;
#ifdef _WIN32
DWORD len, w2;
@@ -10016,15 +10060,15 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
if (rc)
goto leave;
- w2 = txn->mt_next_pgno * env->me_psize;
+ w3 = txn->mt_next_pgno * env->me_psize;
{
mdb_size_t fsize = 0;
if ((rc = mdb_fsize(env->me_fd, &fsize)))
goto leave;
- if (w2 > fsize)
- w2 = fsize;
+ if (w3 > fsize)
+ w3 = fsize;
}
- wsize = w2 - wsize;
+ wsize = w3 - wsize;
while (wsize > 0) {
if (wsize > MAX_WRITE)
w2 = MAX_WRITE;
@@ -10093,7 +10137,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
#ifdef _WIN32
rc = utf8_to_utf16(lpath, -1, &wpath, NULL);
if (rc)
- return rc;
+ goto leave;
newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
free(wpath);
diff --git a/external/db_drivers/liblmdb/midl.h b/external/db_drivers/liblmdb/midl.h
index ed1d75e36..e16aa0c3c 100644
--- a/external/db_drivers/liblmdb/midl.h
+++ b/external/db_drivers/liblmdb/midl.h
@@ -61,7 +61,7 @@ typedef MDB_ID *MDB_IDL;
* limiting factors: sizeof(ID), thread stack size
*/
#ifdef MDB_VL32
-#define MDB_IDL_LOGN 10 /* DB_SIZE is 2^10, UM_SIZE is 2^11 */
+#define MDB_IDL_LOGN 14 /* DB_SIZE is 2^14, UM_SIZE is 2^15 */
#else
#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
#endif
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp
index cdbca52f9..f035bf4c4 100644
--- a/src/blockchain_db/berkeleydb/db_bdb.cpp
+++ b/src/blockchain_db/berkeleydb/db_bdb.cpp
@@ -947,10 +947,12 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags)
LOG_PRINT_RED_L0("Existing BerkeleyDB database was made by a later version. We don't know how it will change yet.");
compatible = false;
}
- else if (VERSION > 0 && result < VERSION)
+#if VERSION > 0
+ else if (result < VERSION)
{
compatible = false;
}
+#endif
}
else
{
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index 0bee8aae9..b43d5742f 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -220,6 +220,13 @@ const std::string lmdb_error(const std::string& error_string, int mdb_res)
} // anonymous namespace
+#define CURSOR(name) \
+ if (!m_cur_ ## name) { \
+ int result = mdb_cursor_open(*m_write_txn, m_ ## name, &m_cur_ ## name); \
+ if (result) \
+ throw0(DB_ERROR(std::string("Failed to open cursor: ").append(mdb_strerror(result)).c_str())); \
+ }
+
namespace cryptonote
{
std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
@@ -428,7 +435,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
- const uint64_t min_increase_size = 128 * (1 << 20);
+ const uint64_t min_increase_size = 512 * (1 << 20);
uint64_t threshold_size = 0;
uint64_t increase_size = 0;
if (batch_num_blocks > 0)
@@ -464,6 +471,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
// batch size estimate * batch safety factor = final size estimate
// Takes into account "reasonable" block size increases in batch.
float batch_safety_factor = 1.7f;
+ float batch_fudge_factor = batch_safety_factor * batch_num_blocks;
// estimate of stored block expanded from raw block, including denormalization and db overhead.
// Note that this probably doesn't grow linearly with block size.
float db_expand_factor = 4.5f;
@@ -485,6 +493,13 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
{
LOG_PRINT_L1("No existing blocks to check for average block size");
}
+ else if (m_cum_count)
+ {
+ avg_block_size = m_cum_size / m_cum_count;
+ LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size);
+ m_cum_size = 0;
+ m_cum_count = 0;
+ }
else
{
for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
@@ -502,8 +517,10 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
avg_block_size = min_block_size;
LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
- threshold_size = avg_block_size * db_expand_factor * batch_num_blocks;
- threshold_size = threshold_size * batch_safety_factor;
+ // bigger safety margin on smaller block sizes
+ if (batch_fudge_factor < 5000.0)
+ batch_fudge_factor = 5000.0;
+ threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor;
return threshold_size;
}
@@ -513,16 +530,16 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
+ CURSOR(block_heights)
MDB_val_copy<crypto::hash> val_h(blk_hash);
- MDB_val unused;
- if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0)
+ if (mdb_cursor_get(m_cur_block_heights, &val_h, NULL, MDB_SET) == 0)
throw1(BLOCK_EXISTS("Attempting to add block that's already in the db"));
if (m_height > 0)
{
MDB_val_copy<crypto::hash> parent_key(blk.prev_id);
MDB_val parent_h;
- if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h))
+ if (mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET))
{
LOG_PRINT_L3("m_height: " << m_height);
LOG_PRINT_L3("parent_key: " << blk.prev_id);
@@ -537,39 +554,48 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
MDB_val_copy<uint64_t> key(m_height);
+ CURSOR(blocks)
+ CURSOR(block_sizes)
+ CURSOR(block_timestamps)
+ CURSOR(block_diffs)
+ CURSOR(block_coins)
+ CURSOR(block_hashes)
+
MDB_val_copy<blobdata> blob(block_to_blob(blk));
- result = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0);
+ result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<size_t> sz(block_size);
- result = mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0);
+ result = mdb_cursor_put(m_cur_block_sizes, &key, &sz, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add block size to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<uint64_t> ts(blk.timestamp);
- result = mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0);
+ result = mdb_cursor_put(m_cur_block_timestamps, &key, &ts, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add block timestamp to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<difficulty_type> diff(cumulative_difficulty);
- result = mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0);
+ result = mdb_cursor_put(m_cur_block_diffs, &key, &diff, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add block cumulative difficulty to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<uint64_t> coinsgen(coins_generated);
- result = mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0);
+ result = mdb_cursor_put(m_cur_block_coins, &key, &coinsgen, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add block total generated coins to db transaction: ").append(mdb_strerror(result)).c_str()));
- result = mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0);
+ result = mdb_cursor_put(m_cur_block_heights, &val_h, &key, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add block height by hash to db transaction: ").append(mdb_strerror(result)).c_str()));
- result = mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0);
+ result = mdb_cursor_put(m_cur_block_hashes, &key, &val_h, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str()));
+ m_cum_size += block_size;
+ m_cum_count++;
}
void BlockchainLMDB::remove_block()
@@ -614,23 +640,27 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr
int result = 0;
+ CURSOR(txs)
+ CURSOR(tx_heights)
+ CURSOR(tx_unlocks)
+
MDB_val_copy<crypto::hash> val_h(tx_hash);
MDB_val unused;
- if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0)
+ if (mdb_cursor_get(m_cur_txs, &val_h, &unused, MDB_SET) == 0)
throw1(TX_EXISTS("Attempting to add transaction that's already in the db"));
MDB_val_copy<blobdata> blob(tx_to_blob(tx));
- result = mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0);
+ result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add tx blob to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<uint64_t> height(m_height);
- result = mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0);
+ result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add tx block height to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<uint64_t> unlock_time(tx.unlock_time);
- result = mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0);
+ result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add tx unlock time to db transaction: ").append(mdb_strerror(result)).c_str()));
}
@@ -668,23 +698,29 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou
int result = 0;
+ CURSOR(output_txs)
+ CURSOR(tx_outputs)
+ CURSOR(output_indices)
+ CURSOR(output_amounts)
+ CURSOR(output_keys)
+
MDB_val_copy<uint64_t> k(m_num_outputs);
MDB_val_copy<crypto::hash> v(tx_hash);
- result = mdb_put(*m_write_txn, m_output_txs, &k, &v, 0);
+ result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add output tx hash to db transaction: ").append(mdb_strerror(result)).c_str()));
- result = mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0);
+ result = mdb_cursor_put(m_cur_tx_outputs, &v, &k, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add <tx hash, global output index> to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<uint64_t> val_local_index(local_index);
- result = mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0);
+ result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND);
if (result)
throw0(DB_ERROR(std::string("Failed to add tx output index to db transaction: ").append(mdb_strerror(result)).c_str()));
MDB_val_copy<uint64_t> val_amount(tx_output.amount);
- result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0);
+ result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &k, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add output amount to db transaction: ").append(mdb_strerror(result)).c_str()));
@@ -697,7 +733,7 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou
MDB_val_copy<output_data_t> data(od);
//MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key);
- if (mdb_put(*m_write_txn, m_output_keys, &k, &data, 0))
+ if (mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND))
throw0(DB_ERROR("Failed to add output pubkey to db transaction"));
}
else
@@ -728,7 +764,7 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transa
}
else
{
- size_t num_elems = 0;
+ mdb_size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP);
@@ -812,7 +848,7 @@ void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uin
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
- size_t num_elems = 0;
+ mdb_size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP);
@@ -855,15 +891,17 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
+ CURSOR(spent_keys)
+
MDB_val_copy<crypto::key_image> val_key(k_image);
MDB_val unused;
- if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0)
+ if (mdb_cursor_get(m_cur_spent_keys, &val_key, &unused, MDB_SET) == 0)
throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db"));
char anything = '\0';
unused.mv_size = sizeof(char);
unused.mv_data = &anything;
- if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0))
+ if (auto result = mdb_cursor_put(m_cur_spent_keys, &val_key, &unused, 0))
throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str()));
}
@@ -945,6 +983,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
m_write_batch_txn = nullptr;
m_batch_active = false;
m_height = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
m_hardfork = nullptr;
}
@@ -1102,10 +1142,12 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
LOG_PRINT_RED_L0("Existing lmdb database was made by a later version. We don't know how it will change yet.");
compatible = false;
}
- else if (VERSION > 0 && *(const uint32_t*)v.mv_data < VERSION)
+#if VERSION > 0
+ else if (*(const uint32_t*)v.mv_data < VERSION)
{
compatible = false;
}
+#endif
}
else
{
@@ -1208,6 +1250,8 @@ void BlockchainLMDB::reset()
txn.commit();
m_height = 0;
m_num_outputs = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
}
std::vector<std::string> BlockchainLMDB::get_filenames() const
@@ -1733,7 +1777,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
else if (result)
throw0(DB_ERROR("DB error attempting to get number of outputs of an amount"));
- size_t num_elems = 0;
+ mdb_size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
TXN_POSTFIX_SUCCESS();
@@ -1829,7 +1873,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash&
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
- size_t num_elems = 0;
+ mdb_size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
@@ -1881,7 +1925,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto:
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
- size_t num_elems = 0;
+ mdb_size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
@@ -2124,6 +2168,8 @@ void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
m_write_batch_txn->m_batch_txn = true;
m_write_txn = m_write_batch_txn;
m_batch_active = true;
+ memset(&m_cursors, 0, sizeof(m_cursors));
+
LOG_PRINT_L3("batch transaction: begin");
}
@@ -2147,6 +2193,7 @@ void BlockchainLMDB::batch_commit()
m_write_txn = nullptr;
delete m_write_batch_txn;
+ memset(&m_cursors, 0, sizeof(m_cursors));
}
void BlockchainLMDB::batch_stop()
@@ -2169,6 +2216,7 @@ void BlockchainLMDB::batch_stop()
delete m_write_batch_txn;
m_write_batch_txn = nullptr;
m_batch_active = false;
+ memset(&m_cursors, 0, sizeof(m_cursors));
LOG_PRINT_L3("batch transaction: end");
}
@@ -2186,6 +2234,7 @@ void BlockchainLMDB::batch_abort()
m_write_batch_txn->abort();
m_batch_active = false;
m_write_batch_txn = nullptr;
+ memset(&m_cursors, 0, sizeof(m_cursors));
LOG_PRINT_L3("batch transaction: aborted");
}
@@ -2217,6 +2266,7 @@ void BlockchainLMDB::block_txn_start()
m_write_txn = nullptr;
throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str()));
}
+ memset(&m_cursors, 0, sizeof(m_cursors));
}
}
@@ -2232,6 +2282,7 @@ void BlockchainLMDB::block_txn_stop()
delete m_write_txn;
m_write_txn = nullptr;
+ memset(&m_cursors, 0, sizeof(m_cursors));
}
}
@@ -2244,6 +2295,7 @@ void BlockchainLMDB::block_txn_abort()
{
delete m_write_txn;
m_write_txn = nullptr;
+ memset(&m_cursors, 0, sizeof(m_cursors));
}
else
{
@@ -2302,6 +2354,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, txn))
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str()));
m_write_txn = &txn;
+ memset(&m_cursors, 0, sizeof(m_cursors));
}
uint64_t num_outputs = m_num_outputs;
@@ -2311,7 +2364,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
if (! m_batch_active)
{
m_write_txn = nullptr;
-
+ memset(&m_cursors, 0, sizeof(m_cursors));
txn.commit();
}
}
@@ -2319,6 +2372,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
{
m_num_outputs = num_outputs;
m_write_txn = nullptr;
+ memset(&m_cursors, 0, sizeof(m_cursors));
throw;
}
@@ -2387,7 +2441,7 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
- size_t num_elems = 0;
+ mdb_size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
if (max <= 1 && num_elems <= max)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
@@ -2421,22 +2475,51 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std
LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index");
break;
}
- while (index >= curcount)
+ if (!curcount && index > num_elems/2)
{
- TIME_MEASURE_START(db1);
- if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0)
+ mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP);
+ mdb_cursor_get(cur, &k, &v, MDB_PREV); /* kludge to unset C_EOF */
+ mdb_cursor_get(cur, &k, &v, MDB_NEXT);
+ mdb_cursor_get(cur, &k, &v, MDB_GET_MULTIPLE);
+
+ curcount = num_elems;
+ while(1)
{
- // allow partial results
- result = false;
- break;
+ TIME_MEASURE_START(db1);
+ int count = v.mv_size / sizeof(uint64_t);
+ curcount -= count;
+ if (curcount > index)
+ {
+ mdb_cursor_get(cur, &k, &v, MDB_PREV_MULTIPLE);
+ } else
+ {
+ blockstart = curcount;
+ curcount += count;
+ break;
+ }
+ TIME_MEASURE_FINISH(db1);
+ t_dbmul += db1;
}
- int count = v.mv_size / sizeof(uint64_t);
-
- blockstart = curcount;
- curcount += count;
- TIME_MEASURE_FINISH(db1);
- t_dbmul += db1;
+ } else
+ {
+ while (index >= curcount)
+ {
+ TIME_MEASURE_START(db1);
+ if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0)
+ {
+ // allow partial results
+ result = false;
+ break;
+ }
+
+ int count = v.mv_size / sizeof(uint64_t);
+
+ blockstart = curcount;
+ curcount += count;
+ TIME_MEASURE_FINISH(db1);
+ t_dbmul += db1;
+ }
}
LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index);
@@ -2473,13 +2556,14 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui
if (global_indices.size() > 0)
{
TXN_PREFIX_RDONLY();
+ lmdb_cur cur(*txn_ptr, m_output_keys);
for (const uint64_t &index : global_indices)
{
MDB_val_copy<uint64_t> k(index);
MDB_val v;
- auto get_result = mdb_get(*txn_ptr, m_output_keys, &k, &v);
+ auto get_result = mdb_cursor_get(cur, &k, &v, MDB_SET);
if (get_result == MDB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist"));
else if (get_result)
@@ -2563,7 +2647,7 @@ void BlockchainLMDB::set_hard_fork_starting_height(uint8_t version, uint64_t hei
MDB_val_copy<uint8_t> val_key(version);
MDB_val_copy<uint64_t> val_value(height);
- if (auto result = mdb_put(*txn_ptr, m_hf_starting_heights, &val_key, &val_value, 0))
+ if (auto result = mdb_put(*txn_ptr, m_hf_starting_heights, &val_key, &val_value, MDB_APPEND))
throw1(DB_ERROR(std::string("Error adding hard fork starting height to db transaction: ").append(mdb_strerror(result)).c_str()));
TXN_BLOCK_POSTFIX_SUCCESS();
@@ -2598,7 +2682,11 @@ void BlockchainLMDB::set_hard_fork_version(uint64_t height, uint8_t version)
MDB_val_copy<uint64_t> val_key(height);
MDB_val_copy<uint8_t> val_value(version);
- if (auto result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0))
+ int result;
+ result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, MDB_APPEND);
+ if (result == MDB_KEYEXIST)
+ result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0);
+ if (result)
throw1(DB_ERROR(std::string("Error adding hard fork version to db transaction: ").append(mdb_strerror(result)).c_str()));
TXN_BLOCK_POSTFIX_SUCCESS();
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index e88bcd01b..150f59475 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -38,6 +38,46 @@
namespace cryptonote
{
+struct mdb_txn_cursors
+{
+ MDB_cursor *m_txc_blocks;
+ MDB_cursor *m_txc_block_heights;
+ MDB_cursor *m_txc_block_hashes;
+ MDB_cursor *m_txc_block_timestamps;
+ MDB_cursor *m_txc_block_sizes;
+ MDB_cursor *m_txc_block_diffs;
+ MDB_cursor *m_txc_block_coins;
+
+ MDB_cursor *m_txc_output_txs;
+ MDB_cursor *m_txc_output_indices;
+ MDB_cursor *m_txc_output_amounts;
+ MDB_cursor *m_txc_output_keys;
+
+ MDB_cursor *m_txc_txs;
+ MDB_cursor *m_txc_tx_heights;
+ MDB_cursor *m_txc_tx_unlocks;
+ MDB_cursor *m_txc_tx_outputs;
+
+ MDB_cursor *m_txc_spent_keys;
+};
+
+#define m_cur_blocks m_cursors.m_txc_blocks
+#define m_cur_block_heights m_cursors.m_txc_block_heights
+#define m_cur_block_hashes m_cursors.m_txc_block_hashes
+#define m_cur_block_timestamps m_cursors.m_txc_block_timestamps
+#define m_cur_block_sizes m_cursors.m_txc_block_sizes
+#define m_cur_block_diffs m_cursors.m_txc_block_diffs
+#define m_cur_block_coins m_cursors.m_txc_block_coins
+#define m_cur_output_txs m_cursors.m_txc_output_txs
+#define m_cur_output_indices m_cursors.m_txc_output_indices
+#define m_cur_output_amounts m_cursors.m_txc_output_amounts
+#define m_cur_output_keys m_cursors.m_txc_output_keys
+#define m_cur_txs m_cursors.m_txc_txs
+#define m_cur_tx_heights m_cursors.m_txc_tx_heights
+#define m_cur_tx_unlocks m_cursors.m_txc_tx_unlocks
+#define m_cur_tx_outputs m_cursors.m_txc_tx_outputs
+#define m_cur_spent_keys m_cursors.m_txc_spent_keys
+
struct mdb_txn_safe
{
mdb_txn_safe();
@@ -306,6 +346,8 @@ private:
uint64_t m_height;
uint64_t m_num_outputs;
+ mutable uint64_t m_cum_size; // used in batch size estimation
+ mutable int m_cum_count;
std::string m_folder;
mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
@@ -313,6 +355,8 @@ private:
bool m_batch_transactions; // support for batch transactions
bool m_batch_active; // whether batch transaction is in progress
+ struct mdb_txn_cursors m_cursors;
+
#if defined(__arm__)
// force a value so it can compile with 32-bit ARM
constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 31;
diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt
index ad2a5b40a..41c3098a0 100644
--- a/src/blockchain_utilities/CMakeLists.txt
+++ b/src/blockchain_utilities/CMakeLists.txt
@@ -99,6 +99,11 @@ target_link_libraries(blockchain_converter
blockchain_db
${CMAKE_THREAD_LIBS_INIT})
+if(${ARCH_WIDTH} EQUAL 32)
+ target_compile_definitions(blockchain_converter
+ PUBLIC -DARCH_WIDTH=32)
+endif()
+
add_dependencies(blockchain_converter
version)
set_property(TARGET blockchain_converter
@@ -117,6 +122,11 @@ target_link_libraries(blockchain_import
p2p
${CMAKE_THREAD_LIBS_INIT})
+if(${ARCH_WIDTH} EQUAL 32)
+ target_compile_definitions(blockchain_import
+ PUBLIC -DARCH_WIDTH=32)
+endif()
+
add_dependencies(blockchain_import
version)
set_property(TARGET blockchain_import
diff --git a/src/blockchain_utilities/blockchain_converter.cpp b/src/blockchain_utilities/blockchain_converter.cpp
index fdd369e79..17b6d81bf 100644
--- a/src/blockchain_utilities/blockchain_converter.cpp
+++ b/src/blockchain_utilities/blockchain_converter.cpp
@@ -57,12 +57,12 @@ bool opt_testnet = false;
// number of blocks per batch transaction
// adjustable through command-line argument according to available RAM
-#if !defined(WIN32)
+#if ARCH_WIDTH != 32
uint64_t db_batch_size_verify = 5000;
#else
// set a lower default batch size for Windows, pending possible LMDB issue with
// large batch size.
-uint64_t db_batch_size_verify = 1000;
+uint64_t db_batch_size_verify = 100;
#endif
// converter only uses verify mode
diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp
index d3045a229..67ad24031 100644
--- a/src/blockchain_utilities/blockchain_import.cpp
+++ b/src/blockchain_utilities/blockchain_import.cpp
@@ -38,6 +38,7 @@
#include "serialization/binary_utils.h" // dump_binary(), parse_binary()
#include "serialization/json_utils.h" // dump_json()
#include "include_base_utils.h"
+#include "blockchain_db/db_types.h"
#include <lmdb.h> // for db flag arguments
@@ -55,11 +56,11 @@ bool opt_testnet = true;
// number of blocks per batch transaction
// adjustable through command-line argument according to available RAM
-#if !defined(WIN32)
+#if ARCH_WIDTH != 32
uint64_t db_batch_size = 20000;
#else
// set a lower default batch size, pending possible LMDB issue with large transaction size
-uint64_t db_batch_size = 1000;
+uint64_t db_batch_size = 100;
#endif
// when verifying, use a smaller default batch size so progress is more
@@ -77,12 +78,48 @@ using namespace cryptonote;
using namespace epee;
-int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags)
+std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
+{
+ std::string result;
+ std::ostringstream s;
+ std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim));
+ result = s.str();
+ if (result.length() > 0)
+ result.erase(result.end()-strlen(delim), result.end());
+ return result;
+}
+
+// db_type: lmdb, berkeley
+// db_mode: safe, fast, fastest
+int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode)
+{
+ uint64_t BDB_FAST_MODE = 0;
+ uint64_t BDB_FASTEST_MODE = 0;
+ uint64_t BDB_SAFE_MODE = 0;
+
+#if defined(BERKELEY_DB)
+ BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC;
+ BDB_FASTEST_MODE = DB_TXN_NOSYNC;
+ BDB_SAFE_MODE = DB_TXN_SYNC;
+#endif
+
+ int db_flags = 0;
+ bool islmdb = db_type == "lmdb";
+ if (db_mode == "safe")
+ db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE;
+ else if (db_mode == "fast")
+ db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE;
+ else if (db_mode == "fastest")
+ db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE;
+ return db_flags;
+}
+
+int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags)
{
std::vector<std::string> db_args;
boost::split(db_args, db_arg_str, boost::is_any_of("#"));
- db_engine = db_args.front();
- boost::algorithm::trim(db_engine);
+ db_type = db_args.front();
+ boost::algorithm::trim(db_type);
if (db_args.size() == 1)
{
@@ -94,28 +131,73 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, in
return 1;
}
+#if !defined(BERKELEY_DB)
+ if (db_type == "berkeley")
+ {
+ LOG_ERROR("BerkeleyDB support disabled.");
+ return false;
+ }
+#endif
+
std::string db_arg_str2 = db_args[1];
boost::split(db_args, db_arg_str2, boost::is_any_of(","));
- for (auto& it : db_args)
+
+ // optionally use a composite mode instead of individual flags
+ const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"};
+ std::string db_mode;
+ if (db_args.size() == 1)
{
- boost::algorithm::trim(it);
- if (it.empty())
- continue;
- LOG_PRINT_L1("LMDB flag: " << it);
- if (it == "nosync")
- mdb_flags |= MDB_NOSYNC;
- else if (it == "nometasync")
- mdb_flags |= MDB_NOMETASYNC;
- else if (it == "writemap")
- mdb_flags |= MDB_WRITEMAP;
- else if (it == "mapasync")
- mdb_flags |= MDB_MAPASYNC;
- else if (it == "nordahead")
- mdb_flags |= MDB_NORDAHEAD;
- else
+ if (db_modes.count(db_args[0]) > 0)
+ {
+ db_mode = db_args[0];
+ }
+ }
+ if (! db_mode.empty())
+ {
+ db_flags = get_db_flags_from_mode(db_type, db_mode);
+ }
+ else
+ {
+ for (auto& it : db_args)
{
- std::cerr << "unrecognized database flag: " << it << ENDL;
- return 1;
+ boost::algorithm::trim(it);
+ if (it.empty())
+ continue;
+ if (db_type == "lmdb")
+ {
+ LOG_PRINT_L1("LMDB flag: " << it);
+ if (it == "nosync")
+ db_flags |= MDB_NOSYNC;
+ else if (it == "nometasync")
+ db_flags |= MDB_NOMETASYNC;
+ else if (it == "writemap")
+ db_flags |= MDB_WRITEMAP;
+ else if (it == "mapasync")
+ db_flags |= MDB_MAPASYNC;
+ else if (it == "nordahead")
+ db_flags |= MDB_NORDAHEAD;
+ else
+ {
+ std::cerr << "unrecognized database flag: " << it << ENDL;
+ return 1;
+ }
+ }
+#if defined(BERKELEY_DB)
+ else if (db_type == "berkeley")
+ {
+ if (it == "txn_write_nosync")
+ db_flags = DB_TXN_WRITE_NOSYNC;
+ else if (it == "txn_nosync")
+ db_flags = DB_TXN_NOSYNC;
+ else if (it == "txn_sync")
+ db_flags = DB_TXN_SYNC;
+ else
+ {
+ std::cerr << "unrecognized database flag: " << it << ENDL;
+ return 1;
+ }
+ }
+#endif
}
}
return 0;
@@ -131,7 +213,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
if (simple_core.support_batch)
use_batch = true;
else
- LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring");
+ LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
}
if (use_batch)
@@ -176,11 +258,11 @@ template <typename FakeCore>
int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0)
{
#if !defined(BLOCKCHAIN_DB)
- static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value,
+ static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_db, FakeCore>::value,
"FakeCore constraint error");
#endif
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
- if (std::is_same<fake_core_lmdb, FakeCore>::value)
+ if (std::is_same<fake_core_db, FakeCore>::value)
{
// Reset stats, in case we're using newly created db, accumulating stats
// from addition of genesis block.
@@ -250,7 +332,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
if (simple_core.support_batch)
use_batch = true;
else
- LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring");
+ LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
}
if (use_batch)
@@ -530,11 +612,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
int main(int argc, char* argv[])
{
#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY)
- std::string default_db_engine = "memory";
+ std::string default_db_type = "memory";
+ std::string default_db_engine_compiled = "memory";
#else
- std::string default_db_engine = "lmdb";
+ std::string default_db_type = "lmdb";
+ std::string default_db_engine_compiled = "blockchain_db";
#endif
+ std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
+ db_types_all.insert("memory");
+
+ std::string available_dbs = join_set_strings(db_types_all, ", ");
+ available_dbs = "available: " + available_dbs;
+
uint32_t log_level = LOG_LEVEL_0;
uint64_t num_blocks = 0;
uint64_t block_stop = 0;
@@ -566,8 +656,7 @@ int main(int argc, char* argv[])
, false
};
const command_line::arg_descriptor<std::string> arg_database = {
- "database", "available: memory, lmdb"
- , default_db_engine
+ "database", available_dbs.c_str(), default_db_type
};
const command_line::arg_descriptor<bool> arg_verify = {"verify",
"Verify blocks and transactions during import", true};
@@ -648,8 +737,6 @@ int main(int argc, char* argv[])
}
}
- std::vector<std::string> db_engines {"memory", "lmdb"};
-
opt_testnet = command_line::get_arg(vm, arg_testnet_on);
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
m_config_folder = command_line::get_arg(vm, data_dir_arg);
@@ -677,23 +764,34 @@ int main(int argc, char* argv[])
}
- std::string db_engine;
- int mdb_flags = 0;
+ std::string db_type;
+ std::string db_engine_compiled;
+ int db_flags = 0;
int res = 0;
- res = parse_db_arguments(db_arg_str, db_engine, mdb_flags);
+ res = parse_db_arguments(db_arg_str, db_type, db_flags);
if (res)
{
std::cerr << "Error parsing database argument(s)" << ENDL;
return 1;
}
- if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end())
+ if (db_types_all.count(db_type) == 0)
{
- std::cerr << "Invalid database engine: " << db_engine << std::endl;
+ std::cerr << "Invalid database type: " << db_type << std::endl;
return 1;
}
- LOG_PRINT_L0("database: " << db_engine);
+ if ((db_type == "lmdb") || (db_type == "berkeley"))
+ {
+ db_engine_compiled = "blockchain_db";
+ }
+ else
+ {
+ db_engine_compiled = "memory";
+ }
+
+ LOG_PRINT_L0("database: " << db_type);
+ LOG_PRINT_L0("database flags: " << db_flags);
LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha);
if (opt_batch)
{
@@ -722,31 +820,31 @@ int main(int argc, char* argv[])
// for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB)
- if (db_engine == "lmdb")
+ if (db_type == "lmdb" || db_type == "berkeley")
{
- fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
+ fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
import_from_file(simple_core, import_file_path, block_stop);
}
- else if (db_engine == "memory")
+ else if (db_type == "memory")
{
fake_core_memory simple_core(m_config_folder, opt_testnet);
import_from_file(simple_core, import_file_path, block_stop);
}
else
{
- std::cerr << "database engine unrecognized" << ENDL;
+ std::cerr << "database type unrecognized" << ENDL;
return 1;
}
// for multi_db_compile:
#else
- if (db_engine != default_db_engine)
+ if (db_engine_compiled != default_db_engine_compiled)
{
- std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl;
+ std::cerr << "Invalid database type for compiled version: " << db_type << std::endl;
return 1;
}
#if BLOCKCHAIN_DB == DB_LMDB
- fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
+ fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
#else
fake_core_memory simple_core(m_config_folder, opt_testnet);
#endif
diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h
index 29f34026d..2fb5031d3 100644
--- a/src/blockchain_utilities/fake_core.h
+++ b/src/blockchain_utilities/fake_core.h
@@ -33,6 +33,9 @@
#include "cryptonote_core/blockchain_storage.h" // in-memory DB
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
+#if defined(BERKELEY_DB)
+#include "blockchain_db/berkeleydb/db_bdb.h"
+#endif
using namespace cryptonote;
@@ -47,7 +50,7 @@ namespace
#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB
-struct fake_core_lmdb
+struct fake_core_db
{
Blockchain m_storage;
HardFork* m_hardfork = nullptr;
@@ -57,15 +60,26 @@ struct fake_core_lmdb
// for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB)
- fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool)
+ fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(&m_storage), m_storage(m_pool)
// for multi_db_compile:
#else
- fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool)
+ fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool)
#endif
{
m_pool.init(path.string());
- BlockchainDB* db = new BlockchainLMDB();
+ BlockchainDB* db = nullptr;
+ if (db_type == "lmdb")
+ db = new BlockchainLMDB();
+#if defined(BERKELEY_DB)
+ else if (db_type == "berkeley")
+ db = new BlockchainBDB();
+#endif
+ else
+ {
+ LOG_ERROR("Attempted to use non-existent database type: " << db_type);
+ throw std::runtime_error("Attempting to use non-existent database type");
+ }
boost::filesystem::path folder(path);
@@ -76,7 +90,7 @@ struct fake_core_lmdb
const std::string filename = folder.string();
try
{
- db->open(filename, mdb_flags);
+ db->open(filename, db_flags);
}
catch (const std::exception& e)
{
@@ -96,7 +110,7 @@ struct fake_core_lmdb
support_batch = true;
support_add_block = true;
}
- ~fake_core_lmdb()
+ ~fake_core_db()
{
m_storage.get_db().check_hard_fork_info();
m_storage.deinit();
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index ed6d3842d..ce829b00f 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -902,6 +902,10 @@ bool wallet2::clear()
m_blockchain.clear();
m_transfers.clear();
m_key_images.clear();
+ m_unconfirmed_txs.clear();
+ m_payments.clear();
+ m_tx_keys.clear();
+ m_confirmed_txs.clear();
m_local_bc_height = 1;
return true;
}
diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h
index f2bbb7346..d0d912cbb 100644
--- a/tests/core_tests/chaingen.h
+++ b/tests/core_tests/chaingen.h
@@ -497,7 +497,7 @@ inline bool do_replay_events(std::vector<test_event_entry>& events)
cryptonote::core c(&pr);
// FIXME: make sure that vm has arg_testnet_on set to true or false if
// this test needs for it to be so.
- const get_test_options<t_test_class> gto;
+ get_test_options<t_test_class> gto;
if (!c.init(vm, &gto.test_options))
{
std::cout << concolor::magenta << "Failed to init core" << concolor::normal << std::endl;
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-29 19:58:36 +0000