aboutsummaryrefslogtreecommitdiff
path: root/src/blockchain_utilities
diff options
context:
space:
mode:
Diffstat (limited to 'src/blockchain_utilities')
-rw-r--r--src/blockchain_utilities/CMakeLists.txt8
-rw-r--r--src/blockchain_utilities/blockchain_export.cpp8
-rw-r--r--src/blockchain_utilities/blockchain_import.cpp98
-rw-r--r--src/blockchain_utilities/blocksdat_file.cpp14
-rw-r--r--src/blockchain_utilities/blocksdat_file.h1
-rw-r--r--src/blockchain_utilities/bootstrap_file.cpp100
-rw-r--r--src/blockchain_utilities/bootstrap_file.h2
7 files changed, 155 insertions, 76 deletions
diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt
index ffdaad4af..bd32e0c55 100644
--- a/src/blockchain_utilities/CMakeLists.txt
+++ b/src/blockchain_utilities/CMakeLists.txt
@@ -77,6 +77,7 @@ target_link_libraries(blockchain_import
cryptonote_core
blockchain_db
p2p
+ version
epee
${Boost_FILESYSTEM_LIBRARY}
${Boost_SYSTEM_LIBRARY}
@@ -89,11 +90,10 @@ if(ARCH_WIDTH)
PUBLIC -DARCH_WIDTH=${ARCH_WIDTH})
endif()
-add_dependencies(blockchain_import
- version)
set_property(TARGET blockchain_import
PROPERTY
OUTPUT_NAME "monero-blockchain-import")
+install(TARGETS blockchain_import DESTINATION bin)
monero_add_executable(blockchain_export
${blockchain_export_sources}
@@ -104,6 +104,7 @@ target_link_libraries(blockchain_export
cryptonote_core
blockchain_db
p2p
+ version
epee
${Boost_FILESYSTEM_LIBRARY}
${Boost_SYSTEM_LIBRARY}
@@ -111,9 +112,8 @@ target_link_libraries(blockchain_export
${CMAKE_THREAD_LIBS_INIT}
${EXTRA_LIBRARIES})
-add_dependencies(blockchain_export
- version)
set_property(TARGET blockchain_export
PROPERTY
OUTPUT_NAME "monero-blockchain-export")
+install(TARGETS blockchain_export DESTINATION bin)
diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp
index 20eca09f2..60672eeda 100644
--- a/src/blockchain_utilities/blockchain_export.cpp
+++ b/src/blockchain_utilities/blockchain_export.cpp
@@ -55,7 +55,7 @@ int main(int argc, char* argv[])
uint64_t block_stop = 0;
bool blocks_dat = false;
- tools::sanitize_locale();
+ tools::on_startup();
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
@@ -109,7 +109,7 @@ int main(int argc, char* argv[])
}
mlog_configure(mlog_get_default_log_path("monero-blockchain-export.log"), true);
- if (!vm["log-level"].defaulted())
+ if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
@@ -178,7 +178,7 @@ int main(int argc, char* argv[])
}
r = core_storage->init(db, opt_testnet);
- CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage");
+ CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
LOG_PRINT_L0("Exporting blockchain raw data...");
@@ -192,7 +192,7 @@ int main(int argc, char* argv[])
BootstrapFile bootstrap;
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
}
- CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data");
+ CHECK_AND_ASSERT_MES(r, 1, "Failed to export blockchain raw data");
LOG_PRINT_L0("Blockchain raw data exported OK");
return 0;
diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp
index 635a70b10..2a956dbdb 100644
--- a/src/blockchain_utilities/blockchain_import.cpp
+++ b/src/blockchain_utilities/blockchain_import.cpp
@@ -169,6 +169,26 @@ int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks,
if (!force && blocks.size() < db_batch_size)
return 0;
+ // wait till we can verify a full HOH without extra, for speed
+ uint64_t new_height = core.get_blockchain_storage().get_db().height() + blocks.size();
+ if (!force && new_height % HASH_OF_HASHES_STEP)
+ return 0;
+
+ std::list<crypto::hash> hashes;
+ for (const auto &b: blocks)
+ {
+ cryptonote::block block;
+ if (!parse_and_validate_block_from_blob(b.block, block))
+ {
+ MERROR("Failed to parse block: "
+ << epee::string_tools::pod_to_hex(get_blob_hash(b.block)));
+ core.cleanup_handle_incoming_blocks();
+ return 1;
+ }
+ hashes.push_back(cryptonote::get_block_hash(block));
+ }
+ core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes);
+
core.prepare_handle_incoming_blocks(blocks);
for(const block_complete_entry& block_entry: blocks)
@@ -230,11 +250,22 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
return false;
}
+ uint64_t start_height = 1, seek_height;
+ if (opt_resume)
+ start_height = core.get_blockchain_storage().get_current_blockchain_height();
+
+ seek_height = start_height;
BootstrapFile bootstrap;
+ streampos pos;
// BootstrapFile bootstrap(import_file_path);
- uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path);
+ uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height);
MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height) total blocks: " << total_source_blocks);
+ if (total_source_blocks-1 <= start_height)
+ {
+ return false;
+ }
+
std::cout << ENDL;
std::cout << "Preparing to read blocks..." << ENDL;
std::cout << ENDL;
@@ -259,11 +290,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
block b;
transaction tx;
int quit = 0;
- uint64_t bytes_read = 0;
-
- uint64_t start_height = 1;
- if (opt_resume)
- start_height = core.get_blockchain_storage().get_current_blockchain_height();
+ uint64_t bytes_read;
// Note that a new blockchain will start with block number 0 (total blocks: 1)
// due to genesis block being added at initialization.
@@ -280,18 +307,35 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
bool use_batch = opt_batch && !opt_verify;
- if (use_batch)
- core.get_blockchain_storage().get_db().batch_start(db_batch_size);
-
MINFO("Reading blockchain from bootstrap file...");
std::cout << ENDL;
std::list<block_complete_entry> blocks;
- // Within the loop, we skip to start_height before we start adding.
- // TODO: Not a bottleneck, but we can use what's done in count_blocks() and
- // only do the chunk size reads, skipping the chunk content reads until we're
- // at start_height.
+ // Skip to start_height before we start adding.
+ {
+ bool q2 = false;
+ import_file.seekg(pos);
+ bytes_read = bootstrap.count_bytes(import_file, start_height-seek_height, h, q2);
+ if (q2)
+ {
+ quit = 2;
+ goto quitting;
+ }
+ h = start_height;
+ }
+
+ if (use_batch)
+ {
+ uint64_t bytes, h2;
+ bool q2;
+ pos = import_file.tellg();
+ bytes = bootstrap.count_bytes(import_file, db_batch_size, h2, q2);
+ if (import_file.eof())
+ import_file.clear();
+ import_file.seekg(pos);
+ core.get_blockchain_storage().get_db().batch_start(db_batch_size, bytes);
+ }
while (! quit)
{
uint32_t chunk_size;
@@ -344,11 +388,6 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
bytes_read += chunk_size;
MDEBUG("Total bytes read: " << bytes_read);
- if (h + NUM_BLOCKS_PER_CHUNK < start_height + 1)
- {
- h += NUM_BLOCKS_PER_CHUNK;
- continue;
- }
if (h > block_stop)
{
std::cout << refresh_string << "block " << h-1
@@ -456,11 +495,16 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
{
if ((h-1) % db_batch_size == 0)
{
+ uint64_t bytes, h2;
+ bool q2;
std::cout << refresh_string;
// zero-based height
std::cout << ENDL << "[- batch commit at height " << h-1 << " -]" << ENDL;
core.get_blockchain_storage().get_db().batch_stop();
- core.get_blockchain_storage().get_db().batch_start(db_batch_size);
+ pos = import_file.tellg();
+ bytes = bootstrap.count_bytes(import_file, db_batch_size, h2, q2);
+ import_file.seekg(pos);
+ core.get_blockchain_storage().get_db().batch_start(db_batch_size, bytes);
std::cout << ENDL;
core.get_blockchain_storage().get_db().show_stats();
}
@@ -477,6 +521,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
}
} // while
+quitting:
import_file.close();
if (opt_verify)
@@ -526,7 +571,7 @@ int main(int argc, char* argv[])
std::string m_config_folder;
std::string db_arg_str;
- tools::sanitize_locale();
+ tools::on_startup();
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
@@ -560,10 +605,7 @@ int main(int argc, char* argv[])
const command_line::arg_descriptor<bool> arg_resume = {"resume",
"Resume from current height if output database already exists", true};
- //command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
- //command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
command_line::add_arg(desc_cmd_sett, arg_input_file);
- //command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_database);
command_line::add_arg(desc_cmd_sett, arg_batch_size);
@@ -609,7 +651,7 @@ int main(int argc, char* argv[])
return 1;
}
- if (! opt_batch && ! vm["batch-size"].defaulted())
+ if (! opt_batch && !command_line::is_arg_defaulted(vm, arg_batch_size))
{
std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL;
return 1;
@@ -619,7 +661,7 @@ int main(int argc, char* argv[])
std::cerr << "Error: batch-size must be > 0" << ENDL;
return 1;
}
- if (opt_verify && vm["batch-size"].defaulted())
+ if (opt_verify && command_line::is_arg_defaulted(vm, arg_batch_size))
{
// usually want batch size default lower if verify on, so progress can be
// frequently saved.
@@ -638,7 +680,7 @@ int main(int argc, char* argv[])
db_arg_str = command_line::get_arg(vm, arg_database);
mlog_configure(mlog_get_default_log_path("monero-blockchain-import.log"), true);
- if (!vm["log-level"].defaulted())
+ if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
@@ -710,7 +752,7 @@ int main(int argc, char* argv[])
}
core.get_blockchain_storage().get_db().set_batch_transactions(true);
- if (! vm["pop-blocks"].defaulted())
+ if (!command_line::is_arg_defaulted(vm, arg_pop_blocks))
{
num_blocks = command_line::get_arg(vm, arg_pop_blocks);
MINFO("height: " << core.get_blockchain_storage().get_current_blockchain_height());
@@ -719,7 +761,7 @@ int main(int argc, char* argv[])
return 0;
}
- if (! vm["drop-hard-fork"].defaulted())
+ if (!command_line::is_arg_defaulted(vm, arg_drop_hf))
{
MINFO("Dropping hard fork tables...");
core.get_blockchain_storage().get_db().drop_hard_fork_info();
diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp
index 63224225f..32e93345e 100644
--- a/src/blockchain_utilities/blocksdat_file.cpp
+++ b/src/blockchain_utilities/blocksdat_file.cpp
@@ -82,7 +82,7 @@ bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64
bool BlocksdatFile::initialize_file(uint64_t block_stop)
{
- const uint32_t nblocks = block_stop + 1;
+ const uint32_t nblocks = (block_stop + 1) / HASH_OF_HASHES_STEP;
unsigned char nblocksc[4];
nblocksc[0] = nblocks & 0xff;
@@ -101,8 +101,16 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop)
void BlocksdatFile::write_block(const crypto::hash& block_hash)
{
- const std::string data(block_hash.data, sizeof(block_hash));
- *m_raw_data_file << data;
+ m_hashes.push_back(block_hash);
+ while (m_hashes.size() >= HASH_OF_HASHES_STEP)
+ {
+ crypto::hash hash;
+ crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
+ memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP * sizeof(crypto::hash), (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
+ m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP);
+ const std::string data(hash.data, sizeof(hash));
+ *m_raw_data_file << data;
+ }
}
bool BlocksdatFile::close()
diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h
index 0a5913058..d43811772 100644
--- a/src/blockchain_utilities/blocksdat_file.h
+++ b/src/blockchain_utilities/blocksdat_file.h
@@ -76,4 +76,5 @@ protected:
private:
uint64_t m_cur_height; // tracks current height during export
+ std::vector<crypto::hash> m_hashes;
};
diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp
index 2b1a5d6c7..a004d3547 100644
--- a/src/blockchain_utilities/bootstrap_file.cpp
+++ b/src/blockchain_utilities/bootstrap_file.cpp
@@ -375,39 +375,15 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file)
return full_header_size;
}
-uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
+uint64_t BootstrapFile::count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit)
{
- boost::filesystem::path raw_file_path(import_file_path);
- boost::system::error_code ec;
- if (!boost::filesystem::exists(raw_file_path, ec))
- {
- MFATAL("bootstrap file not found: " << raw_file_path);
- throw std::runtime_error("Aborting");
- }
- std::ifstream import_file;
- import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
-
- uint64_t h = 0;
- if (import_file.fail())
- {
- MFATAL("import_file.open() fail");
- throw std::runtime_error("Aborting");
- }
-
- uint64_t full_header_size; // 4 byte magic + length of header structures
- full_header_size = seek_to_first_chunk(import_file);
-
- MINFO("Scanning blockchain from bootstrap file...");
- block b;
- bool quit = false;
uint64_t bytes_read = 0;
- int progress_interval = 10;
-
+ uint32_t chunk_size;
+ char buf1[sizeof(chunk_size)];
std::string str1;
- char buf1[2048];
- while (! quit)
+ h = 0;
+ while (1)
{
- uint32_t chunk_size;
import_file.read(buf1, sizeof(chunk_size));
if (!import_file) {
std::cout << refresh_string;
@@ -415,15 +391,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
quit = true;
break;
}
- h += NUM_BLOCKS_PER_CHUNK;
- if ((h-1) % progress_interval == 0)
- {
- std::cout << "\r" << "block height: " << h-1 <<
- " " <<
- std::flush;
- }
bytes_read += sizeof(chunk_size);
-
str1.assign(buf1, sizeof(chunk_size));
if (! ::serialization::parse_binary(str1, chunk_size))
throw std::runtime_error("Error in deserialization of chunk_size");
@@ -456,6 +424,64 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
throw std::runtime_error("Aborting");
}
bytes_read += chunk_size;
+ h += NUM_BLOCKS_PER_CHUNK;
+ if (h >= blocks)
+ break;
+ }
+ return bytes_read;
+}
+
+uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
+{
+ streampos dummy_pos;
+ uint64_t dummy_height = 0;
+ return count_blocks(import_file_path, dummy_pos, dummy_height);
+}
+
+// If seek_height is non-zero on entry, return a stream position <= this height when finished.
+// And return the actual height corresponding to this position. Allows the caller to locate its
+// starting position without having to reread the entire file again.
+uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, streampos &start_pos, uint64_t& seek_height)
+{
+ boost::filesystem::path raw_file_path(import_file_path);
+ boost::system::error_code ec;
+ if (!boost::filesystem::exists(raw_file_path, ec))
+ {
+ MFATAL("bootstrap file not found: " << raw_file_path);
+ throw std::runtime_error("Aborting");
+ }
+ std::ifstream import_file;
+ import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in);
+
+ uint64_t start_height = seek_height;
+ uint64_t h = 0;
+ if (import_file.fail())
+ {
+ MFATAL("import_file.open() fail");
+ throw std::runtime_error("Aborting");
+ }
+
+ uint64_t full_header_size; // 4 byte magic + length of header structures
+ full_header_size = seek_to_first_chunk(import_file);
+
+ MINFO("Scanning blockchain from bootstrap file...");
+ bool quit = false;
+ uint64_t bytes_read = 0, blocks;
+ int progress_interval = 10;
+
+ while (! quit)
+ {
+ if (start_height && h + progress_interval >= start_height - 1)
+ {
+ start_height = 0;
+ start_pos = import_file.tellg();
+ seek_height = h;
+ }
+ bytes_read += count_bytes(import_file, progress_interval, blocks, quit);
+ h += blocks;
+ std::cout << "\r" << "block height: " << h-1 <<
+ " " <<
+ std::flush;
// std::cout << refresh_string;
MDEBUG("Number bytes scanned: " << bytes_read);
diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h
index 1a646b54b..c3969a357 100644
--- a/src/blockchain_utilities/bootstrap_file.h
+++ b/src/blockchain_utilities/bootstrap_file.h
@@ -56,6 +56,8 @@ class BootstrapFile
{
public:
+ uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit);
+ uint64_t count_blocks(const std::string& dir_path, streampos& start_pos, uint64_t& seek_height);
uint64_t count_blocks(const std::string& dir_path);
uint64_t seek_to_first_chunk(std::ifstream& import_file);
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-13 05:05:40 +0000