aboutsummaryrefslogtreecommitdiff
path: root/src/blockchain_utilities/blockchain_import.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/blockchain_utilities/blockchain_import.cpp')
-rw-r--r--src/blockchain_utilities/blockchain_import.cpp454
1 files changed, 170 insertions, 284 deletions
diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp
index a43e2b4ad..635a70b10 100644
--- a/src/blockchain_utilities/blockchain_import.cpp
+++ b/src/blockchain_utilities/blockchain_import.cpp
@@ -32,6 +32,7 @@
#include <fstream>
#include <boost/filesystem.hpp>
+#include "misc_log_ex.h"
#include "bootstrap_file.h"
#include "bootstrap_serialization.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
@@ -39,10 +40,7 @@
#include "serialization/json_utils.h" // dump_json()
#include "include_base_utils.h"
#include "blockchain_db/db_types.h"
-
-#include <lmdb.h> // for db flag arguments
-
-#include "fake_core.h"
+#include "cryptonote_core/cryptonote_core.h"
#undef MONERO_DEFAULT_LOG_CATEGORY
#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
@@ -78,40 +76,16 @@ namespace po = boost::program_options;
using namespace cryptonote;
using namespace epee;
-
-std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
-{
- std::string result;
- std::ostringstream s;
- std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim));
- result = s.str();
- if (result.length() > 0)
- result.erase(result.end()-strlen(delim), result.end());
- return result;
-}
-
-// db_type: lmdb, berkeley
// db_mode: safe, fast, fastest
-int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode)
+int get_db_flags_from_mode(const std::string& db_mode)
{
- uint64_t BDB_FAST_MODE = 0;
- uint64_t BDB_FASTEST_MODE = 0;
- uint64_t BDB_SAFE_MODE = 0;
-
-#if defined(BERKELEY_DB)
- BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC;
- BDB_FASTEST_MODE = DB_TXN_NOSYNC;
- BDB_SAFE_MODE = DB_TXN_SYNC;
-#endif
-
int db_flags = 0;
- bool islmdb = db_type == "lmdb";
if (db_mode == "safe")
- db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE;
+ db_flags = DBF_SAFE;
else if (db_mode == "fast")
- db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE;
+ db_flags = DBF_FAST;
else if (db_mode == "fastest")
- db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE;
+ db_flags = DBF_FASTEST;
return db_flags;
}
@@ -132,14 +106,6 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int&
return 1;
}
-#if !defined(BERKELEY_DB)
- if (db_type == "berkeley")
- {
- MFATAL("BerkeleyDB support disabled.");
- return false;
- }
-#endif
-
std::string db_arg_str2 = db_args[1];
boost::split(db_args, db_arg_str2, boost::is_any_of(","));
@@ -155,70 +121,18 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int&
}
if (! db_mode.empty())
{
- db_flags = get_db_flags_from_mode(db_type, db_mode);
- }
- else
- {
- for (auto& it : db_args)
- {
- boost::algorithm::trim(it);
- if (it.empty())
- continue;
- if (db_type == "lmdb")
- {
- MINFO("LMDB flag: " << it);
- if (it == "nosync")
- db_flags |= MDB_NOSYNC;
- else if (it == "nometasync")
- db_flags |= MDB_NOMETASYNC;
- else if (it == "writemap")
- db_flags |= MDB_WRITEMAP;
- else if (it == "mapasync")
- db_flags |= MDB_MAPASYNC;
- else if (it == "nordahead")
- db_flags |= MDB_NORDAHEAD;
- else
- {
- std::cerr << "unrecognized database flag: " << it << ENDL;
- return 1;
- }
- }
-#if defined(BERKELEY_DB)
- else if (db_type == "berkeley")
- {
- if (it == "txn_write_nosync")
- db_flags = DB_TXN_WRITE_NOSYNC;
- else if (it == "txn_nosync")
- db_flags = DB_TXN_NOSYNC;
- else if (it == "txn_sync")
- db_flags = DB_TXN_SYNC;
- else
- {
- std::cerr << "unrecognized database flag: " << it << ENDL;
- return 1;
- }
- }
-#endif
- }
+ db_flags = get_db_flags_from_mode(db_mode);
}
return 0;
}
-template <typename FakeCore>
-int pop_blocks(FakeCore& simple_core, int num_blocks)
+int pop_blocks(cryptonote::core& core, int num_blocks)
{
- bool use_batch = false;
- if (opt_batch)
- {
- if (simple_core.support_batch)
- use_batch = true;
- else
- MWARNING("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
- }
+ bool use_batch = opt_batch;
if (use_batch)
- simple_core.batch_start();
+ core.get_blockchain_storage().get_db().batch_start();
int quit = 0;
block popped_block;
@@ -226,12 +140,11 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
for (int i=0; i < num_blocks; ++i)
{
// simple_core.m_storage.pop_block_from_blockchain() is private, so call directly through db
- simple_core.m_storage.get_db().pop_block(popped_block, popped_txs);
+ core.get_blockchain_storage().get_db().pop_block(popped_block, popped_txs);
quit = 1;
}
-
if (use_batch)
{
if (quit > 1)
@@ -241,24 +154,74 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
}
else
{
- simple_core.batch_stop();
+ core.get_blockchain_storage().get_db().batch_stop();
}
- simple_core.m_storage.get_db().show_stats();
+ core.get_blockchain_storage().get_db().show_stats();
}
return num_blocks;
}
-template <typename FakeCore>
-int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0)
+int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks, bool force)
{
- if (std::is_same<fake_core_db, FakeCore>::value)
+ if (blocks.empty())
+ return 0;
+ if (!force && blocks.size() < db_batch_size)
+ return 0;
+
+ core.prepare_handle_incoming_blocks(blocks);
+
+ for(const block_complete_entry& block_entry: blocks)
{
- // Reset stats, in case we're using newly created db, accumulating stats
- // from addition of genesis block.
- // This aligns internal db counts with importer counts.
- simple_core.m_storage.get_db().reset_stats();
- }
+ // process transactions
+ for(auto& tx_blob: block_entry.txs)
+ {
+ tx_verification_context tvc = AUTO_VAL_INIT(tvc);
+ core.handle_incoming_tx(tx_blob, tvc, true, true, false);
+ if(tvc.m_verifivation_failed)
+ {
+ MERROR("transaction verification failed, tx_id = "
+ << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)));
+ core.cleanup_handle_incoming_blocks();
+ return 1;
+ }
+ }
+
+ // process block
+
+ block_verification_context bvc = boost::value_initialized<block_verification_context>();
+
+ core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block
+
+ if(bvc.m_verifivation_failed)
+ {
+ MERROR("Block verification failed, id = "
+ << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)));
+ core.cleanup_handle_incoming_blocks();
+ return 1;
+ }
+ if(bvc.m_marked_as_orphaned)
+ {
+ MERROR("Block received at sync phase was marked as orphaned");
+ core.cleanup_handle_incoming_blocks();
+ return 1;
+ }
+
+ } // each download block
+ if (!core.cleanup_handle_incoming_blocks())
+ return 1;
+
+ blocks.clear();
+ return 0;
+}
+
+int import_from_file(cryptonote::core& core, const std::string& import_file_path, uint64_t block_stop=0)
+{
+ // Reset stats, in case we're using newly created db, accumulating stats
+ // from addition of genesis block.
+ // This aligns internal db counts with importer counts.
+ core.get_blockchain_storage().get_db().reset_stats();
+
boost::filesystem::path fs_import_file_path(import_file_path);
boost::system::error_code ec;
if (!boost::filesystem::exists(fs_import_file_path, ec))
@@ -300,7 +263,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
uint64_t start_height = 1;
if (opt_resume)
- start_height = simple_core.m_storage.get_current_blockchain_height();
+ start_height = core.get_blockchain_storage().get_current_blockchain_height();
// Note that a new blockchain will start with block number 0 (total blocks: 1)
// due to genesis block being added at initialization.
@@ -315,21 +278,16 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
MINFO("start block: " << start_height << " stop block: " <<
block_stop);
- bool use_batch = false;
- if (opt_batch)
- {
- if (simple_core.support_batch)
- use_batch = true;
- else
- MWARNING("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
- }
+ bool use_batch = opt_batch && !opt_verify;
if (use_batch)
- simple_core.batch_start(db_batch_size);
+ core.get_blockchain_storage().get_db().batch_start(db_batch_size);
MINFO("Reading blockchain from bootstrap file...");
std::cout << ENDL;
+ std::list<block_complete_entry> blocks;
+
// Within the loop, we skip to start_height before we start adding.
// TODO: Not a bottleneck, but we can use what's done in count_blocks() and
// only do the chunk size reads, skipping the chunk content reads until we're
@@ -359,9 +317,9 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
- if (chunk_size > 100000)
+ if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
{
- MINFO("NOTE: chunk_size " << chunk_size << " > 100000");
+ MINFO("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD);
}
else if (chunk_size == 0) {
MFATAL("ERROR: chunk_size == 0");
@@ -369,9 +327,19 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
}
import_file.read(buffer_block, chunk_size);
if (! import_file) {
- MFATAL("ERROR: unexpected end of file: bytes read before error: "
- << import_file.gcount() << " of chunk_size " << chunk_size);
- return 2;
+ if (import_file.eof())
+ {
+ std::cout << refresh_string;
+ MINFO("End of file reached - file was truncated");
+ quit = 1;
+ break;
+ }
+ else
+ {
+ MFATAL("ERROR: unexpected end of file: bytes read before error: "
+ << import_file.gcount() << " of chunk_size " << chunk_size);
+ return 2;
+ }
}
bytes_read += chunk_size;
MDEBUG("Total bytes read: " << bytes_read);
@@ -424,67 +392,37 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
<< std::flush;
}
- std::vector<transaction> txs;
- std::vector<transaction> archived_txs;
-
- archived_txs = bp.txs;
-
- // std::cout << refresh_string;
- // MDEBUG("txs: " << archived_txs.size());
-
- // if archived_txs is invalid
- // {
- // std::cout << refresh_string;
- // MFATAL("exception while de-archiving txs, height=" << h);
- // quit = 1;
- // break;
- // }
-
- // tx number 1: coinbase tx
- // tx number 2 onwards: archived_txs
- unsigned int tx_num = 1;
- for (transaction tx : archived_txs)
+ if (opt_verify)
{
- ++tx_num;
- // if tx is invalid
- // {
- // MFATAL("exception while indexing tx from txs, height=" << h <<", tx_num=" << tx_num);
- // quit = 1;
- // break;
- // }
-
- // std::cout << refresh_string;
- // MDEBUG("tx hash: " << get_transaction_hash(tx));
-
- // crypto::hash hsh = null_hash;
- // size_t blob_size = 0;
- // NOTE: all tx hashes except for coinbase tx are available in the block data
- // get_transaction_hash(tx, hsh, blob_size);
- // MDEBUG("tx " << tx_num << " " << hsh << " : " << ENDL);
- // MDEBUG(obj_to_json_str(tx) << ENDL);
-
- // add blocks with verification.
- // for Blockchain and blockchain_storage add_new_block().
- if (opt_verify)
+ cryptonote::blobdata block;
+ cryptonote::block_to_blob(bp.block, block);
+ std::list<cryptonote::blobdata> txs;
+ for (const auto &tx: bp.txs)
+ {
+ txs.push_back(cryptonote::blobdata());
+ cryptonote::tx_to_blob(tx, txs.back());
+ }
+ blocks.push_back({block, txs});
+ int ret = check_flush(core, blocks, false);
+ if (ret)
{
- // crypto::hash hsh = null_hash;
- // size_t blob_size = 0;
- // get_transaction_hash(tx, hsh, blob_size);
+ quit = 2; // make sure we don't commit partial block data
+ break;
+ }
+ }
+ else
+ {
+ std::vector<transaction> txs;
+ std::vector<transaction> archived_txs;
+ archived_txs = bp.txs;
- uint8_t version = simple_core.m_storage.get_current_hard_fork_version();
- tx_verification_context tvc = AUTO_VAL_INIT(tvc);
- bool r = true;
- r = simple_core.m_pool.add_tx(tx, tvc, true, true, false, version);
- if (!r)
- {
- MFATAL("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num);
- quit = 1;
- break;
- }
- }
- else
+ // tx number 1: coinbase tx
+ // tx number 2 onwards: archived_txs
+ for (transaction tx : archived_txs)
{
+ // add blocks with verification.
+ // for Blockchain and blockchain_storage add_new_block().
// for add_block() method, without (much) processing.
// don't add coinbase transaction to txs.
//
@@ -493,34 +431,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
// then a for loop for the transactions in txs.
txs.push_back(tx);
}
- }
-
- if (opt_verify)
- {
- block_verification_context bvc = boost::value_initialized<block_verification_context>();
- simple_core.m_storage.add_new_block(b, bvc);
- if (bvc.m_verifivation_failed)
- {
- MFATAL("Failed to add block to blockchain, verification failed, height = " << h);
- MFATAL("skipping rest of file");
- // ok to commit previously batched data because it failed only in
- // verification of potential new block with nothing added to batch
- // yet
- quit = 1;
- break;
- }
- if (! bvc.m_added_to_main_chain)
- {
- MFATAL("Failed to add block to blockchain, height = " << h);
- MFATAL("skipping rest of file");
- // make sure we don't commit partial block data
- quit = 2;
- break;
- }
- }
- else
- {
size_t block_size;
difficulty_type cumulative_difficulty;
uint64_t coins_generated;
@@ -529,14 +440,9 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
cumulative_difficulty = bp.cumulative_difficulty;
coins_generated = bp.coins_generated;
- // std::cout << refresh_string;
- // MDEBUG("block_size: " << block_size);
- // MDEBUG("cumulative_difficulty: " << cumulative_difficulty);
- // MDEBUG("coins_generated: " << coins_generated);
-
try
{
- simple_core.add_block(b, block_size, cumulative_difficulty, coins_generated, txs);
+ core.get_blockchain_storage().get_db().add_block(b, block_size, cumulative_difficulty, coins_generated, txs);
}
catch (const std::exception& e)
{
@@ -545,22 +451,22 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
quit = 2; // make sure we don't commit partial block data
break;
}
- }
- ++num_imported;
- if (use_batch)
- {
- if ((h-1) % db_batch_size == 0)
+ if (use_batch)
{
- std::cout << refresh_string;
- // zero-based height
- std::cout << ENDL << "[- batch commit at height " << h-1 << " -]" << ENDL;
- simple_core.batch_stop();
- simple_core.batch_start(db_batch_size);
- std::cout << ENDL;
- simple_core.m_storage.get_db().show_stats();
+ if ((h-1) % db_batch_size == 0)
+ {
+ std::cout << refresh_string;
+ // zero-based height
+ std::cout << ENDL << "[- batch commit at height " << h-1 << " -]" << ENDL;
+ core.get_blockchain_storage().get_db().batch_stop();
+ core.get_blockchain_storage().get_db().batch_start(db_batch_size);
+ std::cout << ENDL;
+ core.get_blockchain_storage().get_db().show_stats();
+ }
}
}
+ ++num_imported;
}
}
catch (const std::exception& e)
@@ -573,6 +479,13 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
import_file.close();
+ if (opt_verify)
+ {
+ int ret = check_flush(core, blocks, true);
+ if (ret)
+ return ret;
+ }
+
if (use_batch)
{
if (quit > 1)
@@ -582,14 +495,16 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
}
else
{
- simple_core.batch_stop();
+ core.get_blockchain_storage().get_db().batch_stop();
}
- simple_core.m_storage.get_db().show_stats();
- MINFO("Number of blocks imported: " << num_imported);
- if (h > 0)
- // TODO: if there was an error, the last added block is probably at zero-based height h-2
- MINFO("Finished at block: " << h-1 << " total blocks: " << h);
}
+
+ core.get_blockchain_storage().get_db().show_stats();
+ MINFO("Number of blocks imported: " << num_imported);
+ if (h > 0)
+ // TODO: if there was an error, the last added block is probably at zero-based height h-2
+ MINFO("Finished at block: " << h-1 << " total blocks: " << h);
+
std::cout << ENDL;
return 0;
}
@@ -601,12 +516,8 @@ int main(int argc, char* argv[])
epee::string_tools::set_module_name_and_folder(argv[0]);
std::string default_db_type = "lmdb";
- std::string default_db_engine_compiled = "blockchain_db";
- std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
- db_types_all.insert("memory");
-
- std::string available_dbs = join_set_strings(db_types_all, ", ");
+ std::string available_dbs = cryptonote::blockchain_db_types(", ");
available_dbs = "available: " + available_dbs;
uint32_t log_level = 0;
@@ -649,10 +560,10 @@ int main(int argc, char* argv[])
const command_line::arg_descriptor<bool> arg_resume = {"resume",
"Resume from current height if output database already exists", true};
- command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
- command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
+ //command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
+ //command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
command_line::add_arg(desc_cmd_sett, arg_input_file);
- command_line::add_arg(desc_cmd_sett, arg_testnet_on);
+ //command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_database);
command_line::add_arg(desc_cmd_sett, arg_batch_size);
@@ -673,6 +584,7 @@ int main(int argc, char* argv[])
po::options_description desc_options("Allowed options");
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
+ cryptonote::core::init_options(desc_options);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
@@ -751,7 +663,6 @@ int main(int argc, char* argv[])
std::string db_type;
- std::string db_engine_compiled;
int db_flags = 0;
int res = 0;
res = parse_db_arguments(db_arg_str, db_type, db_flags);
@@ -761,25 +672,12 @@ int main(int argc, char* argv[])
return 1;
}
- if (db_types_all.count(db_type) == 0)
+ if (!cryptonote::blockchain_valid_db_type(db_type))
{
std::cerr << "Invalid database type: " << db_type << std::endl;
return 1;
}
- if ((db_type == "lmdb")
-#if defined(BERKELEY_DB)
- || (db_type == "berkeley")
-#endif
- )
- {
- db_engine_compiled = "blockchain_db";
- }
- else
- {
- db_engine_compiled = "memory";
- }
-
MINFO("database: " << db_type);
MINFO("database flags: " << db_flags);
MINFO("verify: " << std::boolalpha << opt_verify << std::noboolalpha);
@@ -798,64 +696,52 @@ int main(int argc, char* argv[])
MINFO("bootstrap file path: " << import_file_path);
MINFO("database path: " << m_config_folder);
+ cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects
+ cryptonote::core core(&pr);
+
try
{
- // fake_core needed for verification to work when enabled.
- //
- // NOTE: don't need fake_core method of doing things when we're going to call
- // BlockchainDB add_block() directly and have available the 3 block
- // properties to do so. Both ways work, but fake core isn't necessary in that
- // circumstance.
-
- if (db_type != "lmdb"
-#if defined(BERKELEY_DB)
- && db_type != "berkeley"
-#endif
- )
+ core.disable_dns_checkpoints(true);
+ if (!core.init(vm, NULL))
{
- std::cerr << "database type unrecognized" << ENDL;
+ std::cerr << "Failed to initialize core" << ENDL;
return 1;
}
-
- // for multi_db_compile:
- fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
+ core.get_blockchain_storage().get_db().set_batch_transactions(true);
if (! vm["pop-blocks"].defaulted())
{
num_blocks = command_line::get_arg(vm, arg_pop_blocks);
- MINFO("height: " << simple_core.m_storage.get_current_blockchain_height());
- pop_blocks(simple_core, num_blocks);
- MINFO("height: " << simple_core.m_storage.get_current_blockchain_height());
+ MINFO("height: " << core.get_blockchain_storage().get_current_blockchain_height());
+ pop_blocks(core, num_blocks);
+ MINFO("height: " << core.get_blockchain_storage().get_current_blockchain_height());
return 0;
}
if (! vm["drop-hard-fork"].defaulted())
{
MINFO("Dropping hard fork tables...");
- simple_core.m_storage.get_db().drop_hard_fork_info();
+ core.get_blockchain_storage().get_db().drop_hard_fork_info();
+ core.deinit();
return 0;
}
- import_from_file(simple_core, import_file_path, block_stop);
+ import_from_file(core, import_file_path, block_stop);
+ // ensure db closed
+ // - transactions properly checked and handled
+ // - disk sync if needed
+ //
+ core.deinit();
}
catch (const DB_ERROR& e)
{
std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL;
+ core.deinit();
return 1;
}
- // destructors called at exit:
- //
- // ensure db closed
- // - transactions properly checked and handled
- // - disk sync if needed
- //
- // fake_core object's destructor is called when it goes out of scope. For an
- // LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn
- // calls delete on its BlockchainDB derived class' object, which closes its
- // files.
return 0;
CATCH_ENTRY("Import error", 1);
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-26 23:05:50 +0000