diff options
| -rw-r--r-- | contrib/epee/include/net/abstract_tcp_server2.inl | 2 | ||||
| -rw-r--r-- | external/db_drivers/liblmdb/lmdb.h | 19 | ||||
| -rw-r--r-- | external/db_drivers/liblmdb/mdb.c | 80 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.cpp | 22 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.h | 2 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_import.cpp | 186 | ||||
| -rw-r--r-- | src/blockchain_utilities/fake_core.h | 24 | ||||
| -rw-r--r-- | tests/core_tests/chaingen.h | 2 |
8 files changed, 259 insertions, 78 deletions
diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 934132ea2..4fb6fa75d 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -144,7 +144,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) // create a random uuid boost::uuids::uuid random_uuid; // that stuff turns out to be included, even though it's from src... Taking advantage - crypto::generate_random_bytes(sizeof(random_uuid), &random_uuid); + random_uuid = crypto::rand<boost::uuids::uuid>(); context.set_details(random_uuid, ip_, remote_ep.port(), is_income); _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << diff --git a/external/db_drivers/liblmdb/lmdb.h b/external/db_drivers/liblmdb/lmdb.h index d616cabf9..a794c9f7d 100644 --- a/external/db_drivers/liblmdb/lmdb.h +++ b/external/db_drivers/liblmdb/lmdb.h @@ -78,6 +78,11 @@ * access to locks and lock file. Exceptions: On read-only filesystems * or with the #MDB_NOLOCK flag described under #mdb_env_open(). * + * - An LMDB configuration will often reserve considerable \b unused + * memory address space and maybe file size for future growth. + * This does not use actual memory or disk space, but users may need + * to understand the difference so they won't be scared off. + * * - By default, in versions before 0.9.10, unused portions of the data * file might receive garbage data from memory freed by other code. * (This does not happen when using the #MDB_WRITEMAP flag.) As of @@ -130,7 +135,7 @@ * * @author Howard Chu, Symas Corporation. * - * @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. + * @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted only as authorized by the OpenLDAP @@ -392,7 +397,9 @@ typedef enum MDB_cursor_op { MDB_PREV_NODUP, /**< Position at last data item of previous key */ MDB_SET, /**< Position at specified key */ MDB_SET_KEY, /**< Position at specified key, return key + data */ - MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ + MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */ + MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to + a page of duplicate data items. Only for #MDB_DUPFIXED */ } MDB_cursor_op; /** @defgroup errors Return Codes @@ -538,9 +545,11 @@ int mdb_env_create(MDB_env **env); * allowed. LMDB will still modify the lock file - except on read-only * filesystems, where LMDB does not use locks. * <li>#MDB_WRITEMAP - * Use a writeable memory map unless MDB_RDONLY is set. This is faster - * and uses fewer mallocs, but loses protection from application bugs + * Use a writeable memory map unless MDB_RDONLY is set. This uses + * fewer mallocs but loses protection from application bugs * like wild pointer writes and other bad updates into the database. + * This may be slightly faster for DBs that fit entirely in RAM, but + * is slower for DBs larger than RAM. * Incompatible with nested transactions. * Do not mix processes with and without MDB_WRITEMAP on the same * environment. This can defeat durability (#mdb_env_sync etc). @@ -1535,7 +1544,7 @@ int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); * <li>EINVAL - cursor is not initialized, or an invalid parameter was specified. * </ul> */ -int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); +int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp); /** @brief Compare two data items according to a particular database. * diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 7be13997a..a366b4024 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -5,7 +5,7 @@ * BerkeleyDB API, but much simplified. */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2016 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -1075,7 +1075,7 @@ typedef struct MDB_db { pgno_t md_branch_pages; /**< number of internal pages */ pgno_t md_leaf_pages; /**< number of leaf pages */ pgno_t md_overflow_pages; /**< number of overflow pages */ - pgno_t md_entries; /**< number of data items */ + mdb_size_t md_entries; /**< number of data items */ pgno_t md_root; /**< the root page of this tree */ } MDB_db; @@ -1426,7 +1426,7 @@ typedef struct MDB_ntxn { #endif /** max bytes to write in one call */ -#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) +#define MAX_WRITE (0x40000000U >> (sizeof(ssize_t) == 4)) /** Check \b txn and \b dbi arguments to a function */ #define TXN_DBI_EXIST(txn, dbi, validity) \ @@ -4160,6 +4160,19 @@ mdb_env_create(MDB_env **env) return MDB_SUCCESS; } +#ifdef _WIN32 +/** @brief Map a result from an NTAPI call to WIN32. */ +static DWORD +mdb_nt2win32(NTSTATUS st) +{ + OVERLAPPED o = {0}; + DWORD br; + o.Internal = st; + GetOverlappedResult(NULL, &o, &br, FALSE); + return GetLastError(); +} +#endif + static int ESECT mdb_env_map(MDB_env *env, void *addr) { @@ -4189,7 +4202,7 @@ mdb_env_map(MDB_env *env, void *addr) rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd); if (rc) - return rc; + return mdb_nt2win32(rc); map = addr; #ifdef MDB_VL32 msize = NUM_METAS * env->me_psize; @@ -4201,7 +4214,7 @@ mdb_env_map(MDB_env *env, void *addr) NtClose(mh); #endif if (rc) - return rc; + return mdb_nt2win32(rc); env->me_map = map; #else #ifdef MDB_VL32 @@ -5256,14 +5269,16 @@ mdb_env_close0(MDB_env *env, int excl) free(env->me_dbflags); free(env->me_path); free(env->me_dirty_list); - free(env->me_txn0); #ifdef MDB_VL32 + if (env->me_txn0 && env->me_txn0->mt_rpages) + free(env->me_txn0->mt_rpages); { unsigned int x; for (x=1; x<=env->me_rpages[0].mid; x++) munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize); } free(env->me_rpages); #endif + free(env->me_txn0); mdb_midl_free(env->me_free_pgs); if (env->me_flags & MDB_ENV_TXKEY) { @@ -5691,7 +5706,8 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret) #define MAP(rc,env,addr,len,off) \ addr = NULL; \ rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \ - len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY) + len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \ + if (rc) rc = mdb_nt2win32(rc) #else off_t off; size_t len; @@ -5765,7 +5781,7 @@ notlocal: pthread_mutex_lock(&env->me_rpmutex); retry: y = 0; - for (i=1; i<tl[0].mid; i++) { + for (i=1; i<=tl[0].mid; i++) { if (!tl[i].mref) { if (!y) y = i; /* tmp overflow pages don't go to env */ @@ -5845,7 +5861,7 @@ retry: if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) { /* purge unref'd pages */ unsigned i, y = 0; - for (i=1; i<el[0].mid; i++) { + for (i=1; i<=el[0].mid; i++) { if (!el[i].mref) { if (!y) y = i; munmap(el[i].mptr, env->me_psize * el[i].mcnt); @@ -6436,8 +6452,10 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) DPRINTF(("cursor_next: top page is %"Y"u in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); - if (mc->mc_flags & C_DEL) + if (mc->mc_flags & C_DEL) { + mc->mc_flags ^= C_DEL; goto skip; + } if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { DPUTS("=====> move to next sibling page"); @@ -6523,6 +6541,8 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); + mc->mc_flags &= ~(C_EOF|C_DEL); + if (mc->mc_ki[mc->mc_top] == 0) { DPUTS("=====> move to prev sibling page"); if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { @@ -6977,6 +6997,30 @@ fetchm: } } break; + case MDB_PREV_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_last(mc, key, data); + else + rc = MDB_SUCCESS; + if (rc == MDB_SUCCESS) { + MDB_cursor *mx = &mc->mc_xcursor->mx_cursor; + if (mx->mc_flags & C_INITIALIZED) { + rc = mdb_cursor_sibling(mx, 0); + if (rc == MDB_SUCCESS) + goto fetchm; + } else { + rc = MDB_NOTFOUND; + } + } + break; case MDB_NEXT: case MDB_NEXT_DUP: case MDB_NEXT_NODUP: @@ -7518,7 +7562,7 @@ new_sub: */ if (do_sub) { int xflags, new_dupdata; - size_t ecount; + mdb_size_t ecount; put_sub: xdata.mv_size = 0; xdata.mv_data = ""; @@ -8233,7 +8277,7 @@ mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) /* Return the count of duplicate data items for the current key */ int -mdb_cursor_count(MDB_cursor *mc, size_t *countp) +mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) { MDB_node *leaf; @@ -9957,7 +10001,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) MDB_txn *txn = NULL; mdb_mutexref_t wmutex = NULL; int rc; - size_t wsize; + mdb_size_t wsize, w3; char *ptr; #ifdef _WIN32 DWORD len, w2; @@ -10016,15 +10060,15 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) if (rc) goto leave; - w2 = txn->mt_next_pgno * env->me_psize; + w3 = txn->mt_next_pgno * env->me_psize; { mdb_size_t fsize = 0; if ((rc = mdb_fsize(env->me_fd, &fsize))) goto leave; - if (w2 > fsize) - w2 = fsize; + if (w3 > fsize) + w3 = fsize; } - wsize = w2 - wsize; + wsize = w3 - wsize; while (wsize > 0) { if (wsize > MAX_WRITE) w2 = MAX_WRITE; @@ -10093,7 +10137,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) #ifdef _WIN32 rc = utf8_to_utf16(lpath, -1, &wpath, NULL); if (rc) - return rc; + goto leave; newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); free(wpath); diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 0bee8aae9..a93079aa7 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -428,7 +428,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size"); - const uint64_t min_increase_size = 128 * (1 << 20); + const uint64_t min_increase_size = 512 * (1 << 20); uint64_t threshold_size = 0; uint64_t increase_size = 0; if (batch_num_blocks > 0) @@ -464,6 +464,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con // batch size estimate * batch safety factor = final size estimate // Takes into account "reasonable" block size increases in batch. float batch_safety_factor = 1.7f; + float batch_fudge_factor = batch_safety_factor * batch_num_blocks; // estimate of stored block expanded from raw block, including denormalization and db overhead. // Note that this probably doesn't grow linearly with block size. float db_expand_factor = 4.5f; @@ -485,6 +486,13 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con { LOG_PRINT_L1("No existing blocks to check for average block size"); } + else if (m_cum_count) + { + avg_block_size = m_cum_size / m_cum_count; + LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size); + m_cum_size = 0; + m_cum_count = 0; + } else { for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) @@ -502,8 +510,10 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con avg_block_size = min_block_size; LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size); - threshold_size = avg_block_size * db_expand_factor * batch_num_blocks; - threshold_size = threshold_size * batch_safety_factor; + // bigger safety margin on smaller block sizes + if (batch_fudge_factor < 5000.0) + batch_fudge_factor = 5000.0; + threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor; return threshold_size; } @@ -570,6 +580,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const if (result) throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str())); + m_cum_size += block_size; + m_cum_count++; } void BlockchainLMDB::remove_block() @@ -945,6 +957,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_write_batch_txn = nullptr; m_batch_active = false; m_height = 0; + m_cum_size = 0; + m_cum_count = 0; m_hardfork = nullptr; } @@ -1208,6 +1222,8 @@ void BlockchainLMDB::reset() txn.commit(); m_height = 0; m_num_outputs = 0; + m_cum_size = 0; + m_cum_count = 0; } std::vector<std::string> BlockchainLMDB::get_filenames() const diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e88bcd01b..1964bfe45 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -306,6 +306,8 @@ private: uint64_t m_height; uint64_t m_num_outputs; + mutable uint64_t m_cum_size; // used in batch size estimation + mutable int m_cum_count; std::string m_folder; mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index d3045a229..daa5db2a3 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -38,6 +38,7 @@ #include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/json_utils.h" // dump_json() #include "include_base_utils.h" +#include "blockchain_db/db_types.h" #include <lmdb.h> // for db flag arguments @@ -77,12 +78,48 @@ using namespace cryptonote; using namespace epee; -int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim) +{ + std::string result; + std::ostringstream s; + std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim)); + result = s.str(); + if (result.length() > 0) + result.erase(result.end()-strlen(delim), result.end()); + return result; +} + +// db_type: lmdb, berkeley +// db_mode: safe, fast, fastest +int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode) +{ + uint64_t BDB_FAST_MODE = 0; + uint64_t BDB_FASTEST_MODE = 0; + uint64_t BDB_SAFE_MODE = 0; + +#if defined(BERKELEY_DB) + BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC; + BDB_FASTEST_MODE = DB_TXN_NOSYNC; + BDB_SAFE_MODE = DB_TXN_SYNC; +#endif + + int db_flags = 0; + bool islmdb = db_type == "lmdb"; + if (db_mode == "safe") + db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE; + else if (db_mode == "fast") + db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE; + else if (db_mode == "fastest") + db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE; + return db_flags; +} + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags) { std::vector<std::string> db_args; boost::split(db_args, db_arg_str, boost::is_any_of("#")); - db_engine = db_args.front(); - boost::algorithm::trim(db_engine); + db_type = db_args.front(); + boost::algorithm::trim(db_type); if (db_args.size() == 1) { @@ -94,28 +131,73 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, in return 1; } +#if !defined(BERKELEY_DB) + if (db_type == "berkeley") + { + LOG_ERROR("BerkeleyDB support disabled."); + return false; + } +#endif + std::string db_arg_str2 = db_args[1]; boost::split(db_args, db_arg_str2, boost::is_any_of(",")); - for (auto& it : db_args) + + // optionally use a composite mode instead of individual flags + const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"}; + std::string db_mode; + if (db_args.size() == 1) { - boost::algorithm::trim(it); - if (it.empty()) - continue; - LOG_PRINT_L1("LMDB flag: " << it); - if (it == "nosync") - mdb_flags |= MDB_NOSYNC; - else if (it == "nometasync") - mdb_flags |= MDB_NOMETASYNC; - else if (it == "writemap") - mdb_flags |= MDB_WRITEMAP; - else if (it == "mapasync") - mdb_flags |= MDB_MAPASYNC; - else if (it == "nordahead") - mdb_flags |= MDB_NORDAHEAD; - else + if (db_modes.count(db_args[0]) > 0) + { + db_mode = db_args[0]; + } + } + if (! db_mode.empty()) + { + db_flags = get_db_flags_from_mode(db_type, db_mode); + } + else + { + for (auto& it : db_args) { - std::cerr << "unrecognized database flag: " << it << ENDL; - return 1; + boost::algorithm::trim(it); + if (it.empty()) + continue; + if (db_type == "lmdb") + { + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + db_flags |= MDB_NOSYNC; + else if (it == "nometasync") + db_flags |= MDB_NOMETASYNC; + else if (it == "writemap") + db_flags |= MDB_WRITEMAP; + else if (it == "mapasync") + db_flags |= MDB_MAPASYNC; + else if (it == "nordahead") + db_flags |= MDB_NORDAHEAD; + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } +#if defined(BERKELEY_DB) + else if (db_type == "berkeley") + { + if (it == "txn_write_nosync") + db_flags = DB_TXN_WRITE_NOSYNC; + else if (it == "txn_nosync") + db_flags = DB_TXN_NOSYNC; + else if (it == "txn_sync") + db_flags = DB_TXN_SYNC; + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } +#endif } } return 0; @@ -131,7 +213,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks) if (simple_core.support_batch) use_batch = true; else - LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring"); } if (use_batch) @@ -176,11 +258,11 @@ template <typename FakeCore> int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0) { #if !defined(BLOCKCHAIN_DB) - static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value, + static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_db, FakeCore>::value, "FakeCore constraint error"); #endif #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) - if (std::is_same<fake_core_lmdb, FakeCore>::value) + if (std::is_same<fake_core_db, FakeCore>::value) { // Reset stats, in case we're using newly created db, accumulating stats // from addition of genesis block. @@ -250,7 +332,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, if (simple_core.support_batch) use_batch = true; else - LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring"); } if (use_batch) @@ -530,11 +612,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path, int main(int argc, char* argv[]) { #if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) - std::string default_db_engine = "memory"; + std::string default_db_type = "memory"; + std::string default_db_engine_compiled = "memory"; #else - std::string default_db_engine = "lmdb"; + std::string default_db_type = "lmdb"; + std::string default_db_engine_compiled = "blockchain_db"; #endif + std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types; + db_types_all.insert("memory"); + + std::string available_dbs = join_set_strings(db_types_all, ", "); + available_dbs = "available: " + available_dbs; + uint32_t log_level = LOG_LEVEL_0; uint64_t num_blocks = 0; uint64_t block_stop = 0; @@ -566,8 +656,7 @@ int main(int argc, char* argv[]) , false }; const command_line::arg_descriptor<std::string> arg_database = { - "database", "available: memory, lmdb" - , default_db_engine + "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<bool> arg_verify = {"verify", "Verify blocks and transactions during import", true}; @@ -648,8 +737,6 @@ int main(int argc, char* argv[]) } } - std::vector<std::string> db_engines {"memory", "lmdb"}; - opt_testnet = command_line::get_arg(vm, arg_testnet_on); auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; m_config_folder = command_line::get_arg(vm, data_dir_arg); @@ -677,23 +764,34 @@ int main(int argc, char* argv[]) } - std::string db_engine; - int mdb_flags = 0; + std::string db_type; + std::string db_engine_compiled; + int db_flags = 0; int res = 0; - res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + res = parse_db_arguments(db_arg_str, db_type, db_flags); if (res) { std::cerr << "Error parsing database argument(s)" << ENDL; return 1; } - if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) + if (db_types_all.count(db_type) == 0) { - std::cerr << "Invalid database engine: " << db_engine << std::endl; + std::cerr << "Invalid database type: " << db_type << std::endl; return 1; } - LOG_PRINT_L0("database: " << db_engine); + if ((db_type == "lmdb") || (db_type == "berkeley")) + { + db_engine_compiled = "blockchain_db"; + } + else + { + db_engine_compiled = "memory"; + } + + LOG_PRINT_L0("database: " << db_type); + LOG_PRINT_L0("database flags: " << db_flags); LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); if (opt_batch) { @@ -722,31 +820,31 @@ int main(int argc, char* argv[]) // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - if (db_engine == "lmdb") + if (db_type == "lmdb" || db_type == "berkeley") { - fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); + fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags); import_from_file(simple_core, import_file_path, block_stop); } - else if (db_engine == "memory") + else if (db_type == "memory") { fake_core_memory simple_core(m_config_folder, opt_testnet); import_from_file(simple_core, import_file_path, block_stop); } else { - std::cerr << "database engine unrecognized" << ENDL; + std::cerr << "database type unrecognized" << ENDL; return 1; } // for multi_db_compile: #else - if (db_engine != default_db_engine) + if (db_engine_compiled != default_db_engine_compiled) { - std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; + std::cerr << "Invalid database type for compiled version: " << db_type << std::endl; return 1; } #if BLOCKCHAIN_DB == DB_LMDB - fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags); + fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags); #else fake_core_memory simple_core(m_config_folder, opt_testnet); #endif diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h index 29f34026d..20a3361b6 100644 --- a/src/blockchain_utilities/fake_core.h +++ b/src/blockchain_utilities/fake_core.h @@ -33,6 +33,7 @@ #include "cryptonote_core/blockchain_storage.h" // in-memory DB #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/berkeleydb/db_bdb.h" using namespace cryptonote; @@ -47,7 +48,7 @@ namespace #if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB -struct fake_core_lmdb +struct fake_core_db { Blockchain m_storage; HardFork* m_hardfork = nullptr; @@ -57,15 +58,26 @@ struct fake_core_lmdb // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) + fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(&m_storage), m_storage(m_pool) // for multi_db_compile: #else - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) + fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool) #endif { m_pool.init(path.string()); - BlockchainDB* db = new BlockchainLMDB(); + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + db = new BlockchainLMDB(); +#if defined(BERKELEY_DB) + else if (db_type == "berkeley") + db = new BlockchainBDB(); +#endif + else + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } boost::filesystem::path folder(path); @@ -76,7 +88,7 @@ struct fake_core_lmdb const std::string filename = folder.string(); try { - db->open(filename, mdb_flags); + db->open(filename, db_flags); } catch (const std::exception& e) { @@ -96,7 +108,7 @@ struct fake_core_lmdb support_batch = true; support_add_block = true; } - ~fake_core_lmdb() + ~fake_core_db() { m_storage.get_db().check_hard_fork_info(); m_storage.deinit(); diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index f2bbb7346..d0d912cbb 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -497,7 +497,7 @@ inline bool do_replay_events(std::vector<test_event_entry>& events) cryptonote::core c(&pr); // FIXME: make sure that vm has arg_testnet_on set to true or false if // this test needs for it to be so. - const get_test_options<t_test_class> gto; + get_test_options<t_test_class> gto; if (!c.init(vm, >o.test_options)) { std::cout << concolor::magenta << "Failed to init core" << concolor::normal << std::endl; |