aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.cpp22
-rw-r--r--src/blockchain_db/lmdb/db_lmdb.h2
-rw-r--r--src/blockchain_utilities/blockchain_import.cpp186
-rw-r--r--src/blockchain_utilities/fake_core.h24
4 files changed, 181 insertions, 53 deletions
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index 0bee8aae9..a93079aa7 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -428,7 +428,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
- const uint64_t min_increase_size = 128 * (1 << 20);
+ const uint64_t min_increase_size = 512 * (1 << 20);
uint64_t threshold_size = 0;
uint64_t increase_size = 0;
if (batch_num_blocks > 0)
@@ -464,6 +464,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
// batch size estimate * batch safety factor = final size estimate
// Takes into account "reasonable" block size increases in batch.
float batch_safety_factor = 1.7f;
+ float batch_fudge_factor = batch_safety_factor * batch_num_blocks;
// estimate of stored block expanded from raw block, including denormalization and db overhead.
// Note that this probably doesn't grow linearly with block size.
float db_expand_factor = 4.5f;
@@ -485,6 +486,13 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
{
LOG_PRINT_L1("No existing blocks to check for average block size");
}
+ else if (m_cum_count)
+ {
+ avg_block_size = m_cum_size / m_cum_count;
+ LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size);
+ m_cum_size = 0;
+ m_cum_count = 0;
+ }
else
{
for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
@@ -502,8 +510,10 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
avg_block_size = min_block_size;
LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
- threshold_size = avg_block_size * db_expand_factor * batch_num_blocks;
- threshold_size = threshold_size * batch_safety_factor;
+ // bigger safety margin on smaller block sizes
+ if (batch_fudge_factor < 5000.0)
+ batch_fudge_factor = 5000.0;
+ threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor;
return threshold_size;
}
@@ -570,6 +580,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
if (result)
throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str()));
+ m_cum_size += block_size;
+ m_cum_count++;
}
void BlockchainLMDB::remove_block()
@@ -945,6 +957,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
m_write_batch_txn = nullptr;
m_batch_active = false;
m_height = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
m_hardfork = nullptr;
}
@@ -1208,6 +1222,8 @@ void BlockchainLMDB::reset()
txn.commit();
m_height = 0;
m_num_outputs = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
}
std::vector<std::string> BlockchainLMDB::get_filenames() const
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index e88bcd01b..1964bfe45 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -306,6 +306,8 @@ private:
uint64_t m_height;
uint64_t m_num_outputs;
+ mutable uint64_t m_cum_size; // used in batch size estimation
+ mutable int m_cum_count;
std::string m_folder;
mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp
index d3045a229..daa5db2a3 100644
--- a/src/blockchain_utilities/blockchain_import.cpp
+++ b/src/blockchain_utilities/blockchain_import.cpp
@@ -38,6 +38,7 @@
#include "serialization/binary_utils.h" // dump_binary(), parse_binary()
#include "serialization/json_utils.h" // dump_json()
#include "include_base_utils.h"
+#include "blockchain_db/db_types.h"
#include <lmdb.h> // for db flag arguments
@@ -77,12 +78,48 @@ using namespace cryptonote;
using namespace epee;
-int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags)
+std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
+{
+ std::string result;
+ std::ostringstream s;
+ std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim));
+ result = s.str();
+ if (result.length() > 0)
+ result.erase(result.end()-strlen(delim), result.end());
+ return result;
+}
+
+// db_type: lmdb, berkeley
+// db_mode: safe, fast, fastest
+int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode)
+{
+ uint64_t BDB_FAST_MODE = 0;
+ uint64_t BDB_FASTEST_MODE = 0;
+ uint64_t BDB_SAFE_MODE = 0;
+
+#if defined(BERKELEY_DB)
+ BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC;
+ BDB_FASTEST_MODE = DB_TXN_NOSYNC;
+ BDB_SAFE_MODE = DB_TXN_SYNC;
+#endif
+
+ int db_flags = 0;
+ bool islmdb = db_type == "lmdb";
+ if (db_mode == "safe")
+ db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE;
+ else if (db_mode == "fast")
+ db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE;
+ else if (db_mode == "fastest")
+ db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE;
+ return db_flags;
+}
+
+int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags)
{
std::vector<std::string> db_args;
boost::split(db_args, db_arg_str, boost::is_any_of("#"));
- db_engine = db_args.front();
- boost::algorithm::trim(db_engine);
+ db_type = db_args.front();
+ boost::algorithm::trim(db_type);
if (db_args.size() == 1)
{
@@ -94,28 +131,73 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, in
return 1;
}
+#if !defined(BERKELEY_DB)
+ if (db_type == "berkeley")
+ {
+ LOG_ERROR("BerkeleyDB support disabled.");
+ return false;
+ }
+#endif
+
std::string db_arg_str2 = db_args[1];
boost::split(db_args, db_arg_str2, boost::is_any_of(","));
- for (auto& it : db_args)
+
+ // optionally use a composite mode instead of individual flags
+ const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"};
+ std::string db_mode;
+ if (db_args.size() == 1)
{
- boost::algorithm::trim(it);
- if (it.empty())
- continue;
- LOG_PRINT_L1("LMDB flag: " << it);
- if (it == "nosync")
- mdb_flags |= MDB_NOSYNC;
- else if (it == "nometasync")
- mdb_flags |= MDB_NOMETASYNC;
- else if (it == "writemap")
- mdb_flags |= MDB_WRITEMAP;
- else if (it == "mapasync")
- mdb_flags |= MDB_MAPASYNC;
- else if (it == "nordahead")
- mdb_flags |= MDB_NORDAHEAD;
- else
+ if (db_modes.count(db_args[0]) > 0)
+ {
+ db_mode = db_args[0];
+ }
+ }
+ if (! db_mode.empty())
+ {
+ db_flags = get_db_flags_from_mode(db_type, db_mode);
+ }
+ else
+ {
+ for (auto& it : db_args)
{
- std::cerr << "unrecognized database flag: " << it << ENDL;
- return 1;
+ boost::algorithm::trim(it);
+ if (it.empty())
+ continue;
+ if (db_type == "lmdb")
+ {
+ LOG_PRINT_L1("LMDB flag: " << it);
+ if (it == "nosync")
+ db_flags |= MDB_NOSYNC;
+ else if (it == "nometasync")
+ db_flags |= MDB_NOMETASYNC;
+ else if (it == "writemap")
+ db_flags |= MDB_WRITEMAP;
+ else if (it == "mapasync")
+ db_flags |= MDB_MAPASYNC;
+ else if (it == "nordahead")
+ db_flags |= MDB_NORDAHEAD;
+ else
+ {
+ std::cerr << "unrecognized database flag: " << it << ENDL;
+ return 1;
+ }
+ }
+#if defined(BERKELEY_DB)
+ else if (db_type == "berkeley")
+ {
+ if (it == "txn_write_nosync")
+ db_flags = DB_TXN_WRITE_NOSYNC;
+ else if (it == "txn_nosync")
+ db_flags = DB_TXN_NOSYNC;
+ else if (it == "txn_sync")
+ db_flags = DB_TXN_SYNC;
+ else
+ {
+ std::cerr << "unrecognized database flag: " << it << ENDL;
+ return 1;
+ }
+ }
+#endif
}
}
return 0;
@@ -131,7 +213,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
if (simple_core.support_batch)
use_batch = true;
else
- LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring");
+ LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
}
if (use_batch)
@@ -176,11 +258,11 @@ template <typename FakeCore>
int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0)
{
#if !defined(BLOCKCHAIN_DB)
- static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value,
+ static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_db, FakeCore>::value,
"FakeCore constraint error");
#endif
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
- if (std::is_same<fake_core_lmdb, FakeCore>::value)
+ if (std::is_same<fake_core_db, FakeCore>::value)
{
// Reset stats, in case we're using newly created db, accumulating stats
// from addition of genesis block.
@@ -250,7 +332,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
if (simple_core.support_batch)
use_batch = true;
else
- LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring");
+ LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
}
if (use_batch)
@@ -530,11 +612,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
int main(int argc, char* argv[])
{
#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY)
- std::string default_db_engine = "memory";
+ std::string default_db_type = "memory";
+ std::string default_db_engine_compiled = "memory";
#else
- std::string default_db_engine = "lmdb";
+ std::string default_db_type = "lmdb";
+ std::string default_db_engine_compiled = "blockchain_db";
#endif
+ std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
+ db_types_all.insert("memory");
+
+ std::string available_dbs = join_set_strings(db_types_all, ", ");
+ available_dbs = "available: " + available_dbs;
+
uint32_t log_level = LOG_LEVEL_0;
uint64_t num_blocks = 0;
uint64_t block_stop = 0;
@@ -566,8 +656,7 @@ int main(int argc, char* argv[])
, false
};
const command_line::arg_descriptor<std::string> arg_database = {
- "database", "available: memory, lmdb"
- , default_db_engine
+ "database", available_dbs.c_str(), default_db_type
};
const command_line::arg_descriptor<bool> arg_verify = {"verify",
"Verify blocks and transactions during import", true};
@@ -648,8 +737,6 @@ int main(int argc, char* argv[])
}
}
- std::vector<std::string> db_engines {"memory", "lmdb"};
-
opt_testnet = command_line::get_arg(vm, arg_testnet_on);
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
m_config_folder = command_line::get_arg(vm, data_dir_arg);
@@ -677,23 +764,34 @@ int main(int argc, char* argv[])
}
- std::string db_engine;
- int mdb_flags = 0;
+ std::string db_type;
+ std::string db_engine_compiled;
+ int db_flags = 0;
int res = 0;
- res = parse_db_arguments(db_arg_str, db_engine, mdb_flags);
+ res = parse_db_arguments(db_arg_str, db_type, db_flags);
if (res)
{
std::cerr << "Error parsing database argument(s)" << ENDL;
return 1;
}
- if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end())
+ if (db_types_all.count(db_type) == 0)
{
- std::cerr << "Invalid database engine: " << db_engine << std::endl;
+ std::cerr << "Invalid database type: " << db_type << std::endl;
return 1;
}
- LOG_PRINT_L0("database: " << db_engine);
+ if ((db_type == "lmdb") || (db_type == "berkeley"))
+ {
+ db_engine_compiled = "blockchain_db";
+ }
+ else
+ {
+ db_engine_compiled = "memory";
+ }
+
+ LOG_PRINT_L0("database: " << db_type);
+ LOG_PRINT_L0("database flags: " << db_flags);
LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha);
if (opt_batch)
{
@@ -722,31 +820,31 @@ int main(int argc, char* argv[])
// for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB)
- if (db_engine == "lmdb")
+ if (db_type == "lmdb" || db_type == "berkeley")
{
- fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
+ fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
import_from_file(simple_core, import_file_path, block_stop);
}
- else if (db_engine == "memory")
+ else if (db_type == "memory")
{
fake_core_memory simple_core(m_config_folder, opt_testnet);
import_from_file(simple_core, import_file_path, block_stop);
}
else
{
- std::cerr << "database engine unrecognized" << ENDL;
+ std::cerr << "database type unrecognized" << ENDL;
return 1;
}
// for multi_db_compile:
#else
- if (db_engine != default_db_engine)
+ if (db_engine_compiled != default_db_engine_compiled)
{
- std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl;
+ std::cerr << "Invalid database type for compiled version: " << db_type << std::endl;
return 1;
}
#if BLOCKCHAIN_DB == DB_LMDB
- fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
+ fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
#else
fake_core_memory simple_core(m_config_folder, opt_testnet);
#endif
diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h
index 29f34026d..20a3361b6 100644
--- a/src/blockchain_utilities/fake_core.h
+++ b/src/blockchain_utilities/fake_core.h
@@ -33,6 +33,7 @@
#include "cryptonote_core/blockchain_storage.h" // in-memory DB
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
+#include "blockchain_db/berkeleydb/db_bdb.h"
using namespace cryptonote;
@@ -47,7 +48,7 @@ namespace
#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB
-struct fake_core_lmdb
+struct fake_core_db
{
Blockchain m_storage;
HardFork* m_hardfork = nullptr;
@@ -57,15 +58,26 @@ struct fake_core_lmdb
// for multi_db_runtime:
#if !defined(BLOCKCHAIN_DB)
- fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool)
+ fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(&m_storage), m_storage(m_pool)
// for multi_db_compile:
#else
- fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool)
+ fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool)
#endif
{
m_pool.init(path.string());
- BlockchainDB* db = new BlockchainLMDB();
+ BlockchainDB* db = nullptr;
+ if (db_type == "lmdb")
+ db = new BlockchainLMDB();
+#if defined(BERKELEY_DB)
+ else if (db_type == "berkeley")
+ db = new BlockchainBDB();
+#endif
+ else
+ {
+ LOG_ERROR("Attempted to use non-existent database type: " << db_type);
+ throw std::runtime_error("Attempting to use non-existent database type");
+ }
boost::filesystem::path folder(path);
@@ -76,7 +88,7 @@ struct fake_core_lmdb
const std::string filename = folder.string();
try
{
- db->open(filename, mdb_flags);
+ db->open(filename, db_flags);
}
catch (const std::exception& e)
{
@@ -96,7 +108,7 @@ struct fake_core_lmdb
support_batch = true;
support_add_block = true;
}
- ~fake_core_lmdb()
+ ~fake_core_db()
{
m_storage.get_db().check_hard_fork_info();
m_storage.deinit();
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-21 08:25:57 +0000