diff options
Diffstat (limited to 'src/blockchain_utilities')
| -rw-r--r-- | src/blockchain_utilities/CMakeLists.txt | 118 | ||||
| -rw-r--r-- | src/blockchain_utilities/README.md | 54 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_converter.cpp | 308 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_export.cpp | 154 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_import.cpp | 770 | ||||
| -rw-r--r-- | src/blockchain_utilities/bootstrap_file.cpp | 501 | ||||
| -rw-r--r-- | src/blockchain_utilities/bootstrap_file.h | 116 | ||||
| -rw-r--r-- | src/blockchain_utilities/bootstrap_serialization.h | 88 | ||||
| -rw-r--r-- | src/blockchain_utilities/fake_core.h | 165 |
9 files changed, 2274 insertions, 0 deletions
diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt new file mode 100644 index 000000000..5be37c450 --- /dev/null +++ b/src/blockchain_utilities/CMakeLists.txt @@ -0,0 +1,118 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(blockchain_converter_sources + blockchain_converter.cpp + ) + +set(blockchain_converter_private_headers) + +bitmonero_private_headers(blockchain_converter + ${blockchain_converter_private_headers}) + +set(blockchain_import_sources + blockchain_import.cpp + bootstrap_file.cpp + ) + +set(blockchain_import_private_headers + fake_core.h + bootstrap_file.h + bootstrap_serialization.h + ) + +bitmonero_private_headers(blockchain_import + ${blockchain_import_private_headers}) + +set(blockchain_export_sources + blockchain_export.cpp + bootstrap_file.cpp + ) + +set(blockchain_export_private_headers + bootstrap_file.h + bootstrap_serialization.h + ) + +bitmonero_private_headers(blockchain_export + ${blockchain_export_private_headers}) + + + +if (BLOCKCHAIN_DB STREQUAL DB_LMDB) +bitmonero_add_executable(blockchain_converter + ${blockchain_converter_sources} + ${blockchain_converter_private_headers}) + +target_link_libraries(blockchain_converter + LINK_PRIVATE + cryptonote_core + p2p + blockchain_db + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_converter + version) +set_property(TARGET blockchain_converter + PROPERTY + OUTPUT_NAME "blockchain_converter") +endif () + +bitmonero_add_executable(blockchain_import + ${blockchain_import_sources} + ${blockchain_import_private_headers}) + +target_link_libraries(blockchain_import + LINK_PRIVATE + cryptonote_core + blockchain_db + p2p + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_import + version) +set_property(TARGET blockchain_import + PROPERTY + OUTPUT_NAME "blockchain_import") + +bitmonero_add_executable(blockchain_export + ${blockchain_export_sources} + ${blockchain_export_private_headers}) + +target_link_libraries(blockchain_export + LINK_PRIVATE + cryptonote_core + p2p + blockchain_db + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_export + version) +set_property(TARGET blockchain_export + PROPERTY + OUTPUT_NAME "blockchain_export") diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md new file mode 100644 index 000000000..ecf8a7c42 --- /dev/null +++ b/src/blockchain_utilities/README.md @@ -0,0 +1,54 @@ + +For importing into the LMDB database, compile with `DATABASE=lmdb` + +e.g. + +`DATABASE=lmdb make release` + +This is also the default compile setting on the blockchain branch. + +By default, the exporter will use the original in-memory database (blockchain.bin) as its source. +This default is to make migrating to an LMDB database easy, without having to recompile anything. +To change the source, adjust `SOURCE_DB` in `src/blockchain_utilities/bootstrap_file.h` according to the comments. + +# Usage: + +See also each utility's "--help" option. + +## Export an existing in-memory database + +`$ blockchain_export` + +This loads the existing blockchain, for whichever database type it was compiled for, and exports it to `$MONERO_DATA_DIR/export/blockchain.raw` + +## Import the exported file + +`$ blockchain_import` + +This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database. + +Defaults: `--batch on`, `--batch size 20000`, `--verify on` + +Batch size refers to number of blocks and can be adjusted for performance based on available RAM. + +Verification should only be turned off if importing from a trusted blockchain. + +```bash +# use default settings to import blockchain.raw into database +$ blockchain_import + +# fast import with large batch size, verification off +$ blockchain_import --batch-size 100000 --verify off + +# LMDB flags can be set by appending them to the database type: +# flags: nosync, nometasync, writemap, mapasync +$ blockchain_import --database lmdb#nosync +$ blockchain_import --database lmdb#nosync,nometasync +``` + +## Blockchain converter with batching +`blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate. + +```bash +$ blockchain_converter --batch on --batch-size 20000 +``` diff --git a/src/blockchain_utilities/blockchain_converter.cpp b/src/blockchain_utilities/blockchain_converter.cpp new file mode 100644 index 000000000..855dde644 --- /dev/null +++ b/src/blockchain_utilities/blockchain_converter.cpp @@ -0,0 +1,308 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "common/util.h" +#include "warnings.h" +#include "crypto/crypto.h" +#include "cryptonote_config.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "misc_language.h" +#include "cryptonote_core/blockchain_storage.h" +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "cryptonote_core/tx_pool.h" +#include "common/command_line.h" +#include "serialization/json_utils.h" +#include "include_base_utils.h" +#include "version.h" +#include <iostream> + + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace +{ + +// CONFIG +bool opt_batch = true; +bool opt_resume = true; +bool opt_testnet = false; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +#if !defined(WIN32) +uint64_t db_batch_size_verify = 5000; +#else +// set a lower default batch size for Windows, pending possible LMDB issue with +// large batch size. +uint64_t db_batch_size_verify = 1000; +#endif + +// converter only uses verify mode +uint64_t db_batch_size = db_batch_size_verify; + +} + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +struct fake_core +{ + Blockchain dummy; + tx_memory_pool m_pool; + + blockchain_storage m_storage; + +#if !defined(BLOCKCHAIN_DB) + // for multi_db_runtime: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(&dummy), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(dummy), m_storage(&m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + } +}; + +int main(int argc, char* argv[]) +{ + uint64_t height = 0; + uint64_t start_block = 0; + uint64_t end_block = 0; + uint64_t num_blocks = 0; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", LOG_LEVEL_0}; + const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , opt_testnet + }; + const command_line::arg_descriptor<uint64_t> arg_block_number = + {"block-number", "Number of blocks (default: use entire source blockchain)", + 0}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_block_number); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + const command_line::arg_descriptor<bool> arg_batch = {"batch", + "Batch transactions for faster import", true}; + const command_line::arg_descriptor<bool> arg_resume = {"resume", + "Resume from current height if output database already exists", true}; + + // call add_options() directly for these arguments since command_line helpers + // support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + + return true; + }); + if (!r) + return 1; + + int log_level = command_line::get_arg(vm, arg_log_level); + opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + return 1; + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + return 1; + } + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + + std::string src_folder; + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + src_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path dest_folder(src_folder); + + num_blocks = command_line::get_arg(vm, arg_block_number); + + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + fake_core c(src_folder, opt_testnet); + + height = c.m_storage.get_current_blockchain_height(); + + BlockchainDB *blockchain; + blockchain = new BlockchainLMDB(opt_batch); + dest_folder /= blockchain->get_db_name(); + LOG_PRINT_L0("Source blockchain: " << src_folder); + LOG_PRINT_L0("Dest blockchain: " << dest_folder.string()); + LOG_PRINT_L0("Opening dest blockchain (BlockchainDB " << blockchain->get_db_name() << ")"); + blockchain->open(dest_folder.string()); + LOG_PRINT_L0("Source blockchain height: " << height); + LOG_PRINT_L0("Dest blockchain height: " << blockchain->height()); + + if (opt_resume) + // next block number to add is same as current height + start_block = blockchain->height(); + + if (! num_blocks || (start_block + num_blocks > height)) + end_block = height - 1; + else + end_block = start_block + num_blocks - 1; + + LOG_PRINT_L0("start height: " << start_block+1 << " stop height: " << + end_block+1); + + if (start_block > end_block) + { + LOG_PRINT_L0("Finished: no blocks to add"); + delete blockchain; + return 0; + } + + if (opt_batch) + blockchain->batch_start(); + uint64_t i = 0; + for (i = start_block; i < end_block + 1; ++i) + { + // block: i height: i+1 end height: end_block + 1 + if ((i+1) % 10 == 0) + { + std::cout << "\r \r" << "height " << i+1 << "/" << + end_block+1 << " (" << (i+1)*100/(end_block+1)<< "%)" << std::flush; + } + // for debugging: + // std::cout << "height " << i+1 << "/" << end_block+1 + // << " ((" << i+1 << ")*100/(end_block+1))" << "%)" << ENDL; + + block b = c.m_storage.get_block(i); + size_t bsize = c.m_storage.get_block_size(i); + difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); + uint64_t bcoins = c.m_storage.get_block_coins_generated(i); + std::vector<transaction> txs; + std::vector<crypto::hash> missed; + + c.m_storage.get_transactions(b.tx_hashes, txs, missed); + if (missed.size()) + { + std::cout << ENDL; + std::cerr << "Missed transaction(s) for block at height " << i + 1 << ", exiting" << ENDL; + delete blockchain; + return 1; + } + + try + { + blockchain->add_block(b, bsize, bdiff, bcoins, txs); + + if (opt_batch) + { + if ((i < end_block) && ((i + 1) % db_batch_size == 0)) + { + std::cout << "\r \r"; + std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL; + blockchain->batch_stop(); + blockchain->batch_start(); + std::cout << ENDL; + blockchain->show_stats(); + } + } + } + catch (const std::exception& e) + { + std::cout << ENDL; + std::cerr << "Error adding block " << i << " to new blockchain: " << e.what() << ENDL; + delete blockchain; + return 2; + } + } + if (opt_batch) + { + std::cout << "\r \r" << "height " << i << "/" << + end_block+1 << " (" << (i)*100/(end_block+1)<< "%)" << std::flush; + std::cout << ENDL; + std::cout << "[- batch commit at height " << i << " -]" << ENDL; + blockchain->batch_stop(); + } + std::cout << ENDL; + blockchain->show_stats(); + std::cout << "Finished at height: " << i << " block: " << i-1 << ENDL; + + delete blockchain; + return 0; +} diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp new file mode 100644 index 000000000..aa34ea1dc --- /dev/null +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -0,0 +1,154 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "bootstrap_file.h" +#include "common/command_line.h" +#include "version.h" + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace po = boost::program_options; +using namespace epee; // log_space + +int main(int argc, char* argv[]) +{ + uint32_t log_level = 0; + uint64_t block_height = 0; + std::string import_filename = BLOCKCHAIN_RAW; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor<uint64_t> arg_block_height = {"block-number", "", block_height}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_block_height); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + log_level = command_line::get_arg(vm, arg_log_level); + block_height = command_line::get_arg(vm, arg_block_height); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + bool opt_testnet = command_line::get_arg(vm, arg_testnet_on); + + std::string m_config_folder; + + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + m_config_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path output_dir {m_config_folder}; + output_dir /= "export"; + LOG_PRINT_L0("Export directory: " << output_dir.string()); + + // If we wanted to use the memory pool, we would set up a fake_core. + +#if SOURCE_DB == DB_MEMORY + // blockchain_storage* core_storage = NULL; + // tx_memory_pool m_mempool(*core_storage); // is this fake anyway? just passing in NULL! so m_mempool can't be used anyway, right? + // core_storage = new blockchain_storage(&m_mempool); + + blockchain_storage* core_storage = new blockchain_storage(NULL); + LOG_PRINT_L0("Initializing source blockchain (in-memory database)"); + r = core_storage->init(m_config_folder, opt_testnet); +#else + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + Blockchain* core_storage = NULL; + tx_memory_pool m_mempool(*core_storage); + core_storage = new Blockchain(m_mempool); + + BlockchainDB* db = new BlockchainLMDB(); + boost::filesystem::path folder(m_config_folder); + folder /= db->get_db_name(); + LOG_PRINT_L0("Loading blockchain from folder " << folder.string() << " ..."); + const std::string filename = folder.string(); + try + { + db->open(filename); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + throw; + } + r = core_storage->init(db, opt_testnet); +#endif + + CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + LOG_PRINT_L0("Exporting blockchain raw data..."); + + BootstrapFile bootstrap; + r = bootstrap.store_blockchain_raw(core_storage, NULL, output_dir, block_height); + CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data"); + LOG_PRINT_L0("Blockchain raw data exported OK"); +} diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp new file mode 100644 index 000000000..6777cc8fb --- /dev/null +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -0,0 +1,770 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <atomic> +#include <cstdio> +#include <algorithm> +#include <fstream> + +#include <boost/filesystem.hpp> +#include "bootstrap_file.h" +#include "bootstrap_serialization.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "serialization/binary_utils.h" // dump_binary(), parse_binary() +#include "serialization/json_utils.h" // dump_json() +#include "include_base_utils.h" + +#include <lmdb.h> // for db flag arguments + +#include "fake_core.h" + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace +{ +// CONFIG +bool opt_batch = true; +bool opt_verify = true; // use add_new_block, which does verification before calling add_block +bool opt_resume = true; +bool opt_testnet = true; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +#if !defined(WIN32) +uint64_t db_batch_size = 20000; +#else +// set a lower default batch size, pending possible LMDB issue with large transaction size +uint64_t db_batch_size = 1000; +#endif + +// when verifying, use a smaller default batch size so progress is more +// frequently saved +uint64_t db_batch_size_verify = 5000; + +std::string refresh_string = "\r \r"; +} + + + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +{ + std::vector<std::string> db_args; + boost::split(db_args, db_arg_str, boost::is_any_of("#")); + db_engine = db_args.front(); + boost::algorithm::trim(db_engine); + + if (db_args.size() == 1) + { + return 0; + } + else if (db_args.size() > 2) + { + std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL; + return 1; + } + + std::string db_arg_str2 = db_args[1]; + boost::split(db_args, db_arg_str2, boost::is_any_of(",")); + for (auto& it : db_args) + { + boost::algorithm::trim(it); + if (it.empty()) + continue; + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + mdb_flags |= MDB_NOSYNC; + else if (it == "nometasync") + mdb_flags |= MDB_NOMETASYNC; + else if (it == "writemap") + mdb_flags |= MDB_WRITEMAP; + else if (it == "mapasync") + mdb_flags |= MDB_MAPASYNC; + else if (it == "nordahead") + mdb_flags |= MDB_NORDAHEAD; + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } + return 0; +} + + +template <typename FakeCore> +int pop_blocks(FakeCore& simple_core, int num_blocks) +{ + bool use_batch = false; + if (opt_batch) + { + if (simple_core.support_batch) + use_batch = true; + else + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + } + + if (use_batch) + simple_core.batch_start(); + + int quit = 0; + block popped_block; + std::vector<transaction> popped_txs; + for (int i=0; i < num_blocks; ++i) + { + // simple_core.m_storage.pop_block_from_blockchain() is private, so call directly through db + simple_core.m_storage.get_db().pop_block(popped_block, popped_txs); + quit = 1; + } + + + + if (use_batch) + { + if (quit > 1) + { + // There was an error, so don't commit pending data. + // Destructor will abort write txn. + } + else + { + simple_core.batch_stop(); + } +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db().show_stats(); +#endif + } + + return num_blocks; +} + +template <typename FakeCore> +int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint64_t stop_height=0) +{ +#if !defined(BLOCKCHAIN_DB) + static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value, + "FakeCore constraint error"); +#endif +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + if (std::is_same<fake_core_lmdb, FakeCore>::value) + { + // Reset stats, in case we're using newly created db, accumulating stats + // from addition of genesis block. + // This aligns internal db counts with importer counts. + simple_core.m_storage.get_db().reset_stats(); + } +#endif + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("bootstrap file not found: " << raw_file_path); + return false; + } + + BootstrapFile bootstrap; + // BootstrapFile bootstrap(import_file_path); + uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path); + LOG_PRINT_L0("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height) total blocks: " << total_source_blocks); + + std::cout << ENDL; + std::cout << "Preparing to read blocks..." << ENDL; + std::cout << ENDL; + + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + uint64_t num_imported = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + return false; + } + + // 4 byte magic + (currently) 1024 byte header structures + bootstrap.seek_to_first_chunk(import_file); + + std::string str1; + char buffer1[1024]; + char buffer_block[BUFFER_SIZE]; + block b; + transaction tx; + int quit = 0; + uint64_t bytes_read = 0; + + uint64_t start_height = 1; + if (opt_resume) + start_height = simple_core.m_storage.get_current_blockchain_height(); + + // Note that a new blockchain will start with block number 0 (total blocks: 1) + // due to genesis block being added at initialization. + + if (! stop_height) + { + stop_height = total_source_blocks - 1; + } + + // These are what we'll try to use, and they don't have to be a determination + // from source and destination blockchains, but those are the defaults. + LOG_PRINT_L0("start block: " << start_height << " stop block: " << + stop_height); + + bool use_batch = false; + if (opt_batch) + { + if (simple_core.support_batch) + use_batch = true; + else + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + } + + if (use_batch) + simple_core.batch_start(); + + LOG_PRINT_L0("Reading blockchain from bootstrap file..."); + std::cout << ENDL; + + // Within the loop, we skip to start_height before we start adding. + // TODO: Not a bottleneck, but we can use what's done in count_blocks() and + // only do the chunk size reads, skipping the chunk content reads until we're + // at start_height. + while (! quit) + { + uint32_t chunk_size; + import_file.read(buffer1, sizeof(chunk_size)); + // TODO: bootstrap.read_chunk(); + if (! import_file) { + std::cout << refresh_string; + LOG_PRINT_L0("End of file reached"); + quit = 1; + break; + } + bytes_read += sizeof(chunk_size); + + str1.assign(buffer1, sizeof(chunk_size)); + if (! ::serialization::parse_binary(str1, chunk_size)) + { + throw std::runtime_error("Error in deserialization of chunk size"); + } + LOG_PRINT_L1("chunk_size: " << chunk_size); + + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + LOG_PRINT_L0("NOTE: chunk_size " << chunk_size << " > 100000"); + } + else if (chunk_size < 0) { + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " < 0"); + return 2; + } + import_file.read(buffer_block, chunk_size); + if (! import_file) { + LOG_PRINT_L0("ERROR: unexpected end of file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + return 2; + } + bytes_read += chunk_size; + LOG_PRINT_L3("Total bytes read: " << bytes_read); + + if (h + NUM_BLOCKS_PER_CHUNK < start_height + 1) + { + h += NUM_BLOCKS_PER_CHUNK; + continue; + } + if (h > stop_height) + { + std::cout << refresh_string << "block " << h-1 + << " / " << stop_height + << std::flush; + std::cout << ENDL << ENDL; + LOG_PRINT_L0("Specified block number reached - stopping. block: " << h-1 << " total blocks: " << h); + quit = 1; + break; + } + + try + { + str1.assign(buffer_block, chunk_size); + bootstrap::block_package bp; + if (! ::serialization::parse_binary(str1, bp)) + throw std::runtime_error("Error in deserialization of chunk"); + + int display_interval = 1000; + int progress_interval = 10; + // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported. + for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; ++chunk_ind) + { + ++h; + if ((h-1) % display_interval == 0) + { + std::cout << refresh_string; + LOG_PRINT_L0("loading block number " << h-1); + } + else + { + LOG_PRINT_L3("loading block number " << h-1); + } + b = bp.block; + LOG_PRINT_L2("block prev_id: " << b.prev_id << ENDL); + + if ((h-1) % progress_interval == 0) + { + std::cout << refresh_string << "block " << h-1 + << " / " << stop_height + << std::flush; + } + + std::vector<transaction> txs; + std::vector<transaction> archived_txs; + + archived_txs = bp.txs; + + // std::cout << refresh_string; + // LOG_PRINT_L1("txs: " << archived_txs.size()); + + // if archived_txs is invalid + // { + // std::cout << refresh_string; + // LOG_PRINT_RED_L0("exception while de-archiving txs, height=" << h); + // quit = 1; + // break; + // } + + // tx number 1: coinbase tx + // tx number 2 onwards: archived_txs + unsigned int tx_num = 1; + for (const transaction& tx : archived_txs) + { + ++tx_num; + // if tx is invalid + // { + // LOG_PRINT_RED_L0("exception while indexing tx from txs, height=" << h <<", tx_num=" << tx_num); + // quit = 1; + // break; + // } + + // std::cout << refresh_string; + // LOG_PRINT_L1("tx hash: " << get_transaction_hash(tx)); + + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // NOTE: all tx hashes except for coinbase tx are available in the block data + // get_transaction_hash(tx, hsh, blob_size); + // LOG_PRINT_L0("tx " << tx_num << " " << hsh << " : " << ENDL); + // LOG_PRINT_L0(obj_to_json_str(tx) << ENDL); + + // add blocks with verification. + // for Blockchain and blockchain_storage add_new_block(). + if (opt_verify) + { + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // get_transaction_hash(tx, hsh, blob_size); + tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool r = true; + r = simple_core.m_pool.add_tx(tx, tvc, true); + if (!r) + { + LOG_PRINT_RED_L0("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num); + quit = 1; + break; + } + } + else + { + // for add_block() method, without (much) processing. + // don't add coinbase transaction to txs. + // + // because add_block() calls + // add_transaction(blk_hash, blk.miner_tx) first, and + // then a for loop for the transactions in txs. + txs.push_back(tx); + } + } + + if (opt_verify) + { + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + simple_core.m_storage.add_new_block(b, bvc); + + if (bvc.m_verifivation_failed) + { + LOG_PRINT_L0("Failed to add block to blockchain, verification failed, height = " << h); + LOG_PRINT_L0("skipping rest of file"); + // ok to commit previously batched data because it failed only in + // verification of potential new block with nothing added to batch + // yet + quit = 1; + break; + } + if (! bvc.m_added_to_main_chain) + { + LOG_PRINT_L0("Failed to add block to blockchain, height = " << h); + LOG_PRINT_L0("skipping rest of file"); + // make sure we don't commit partial block data + quit = 2; + break; + } + } + else + { + size_t block_size; + difficulty_type cumulative_difficulty; + uint64_t coins_generated; + + block_size = bp.block_size; + cumulative_difficulty = bp.cumulative_difficulty; + coins_generated = bp.coins_generated; + + // std::cout << refresh_string; + // LOG_PRINT_L2("block_size: " << block_size); + // LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty); + // LOG_PRINT_L2("coins_generated: " << coins_generated); + + try + { + simple_core.add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what()); + quit = 2; // make sure we don't commit partial block data + break; + } + } + ++num_imported; + + if (use_batch) + { + if ((h-1) % db_batch_size == 0) + { + std::cout << refresh_string; + // zero-based height + std::cout << ENDL << "[- batch commit at height " << h-1 << " -]" << ENDL; + simple_core.batch_stop(); + simple_core.batch_start(); + std::cout << ENDL; +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db().show_stats(); +#endif + } + } + } + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("exception while reading from file, height=" << h); + return 2; + } + } // while + + import_file.close(); + + if (use_batch) + { + if (quit > 1) + { + // There was an error, so don't commit pending data. + // Destructor will abort write txn. + } + else + { + simple_core.batch_stop(); + } +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db().show_stats(); +#endif + LOG_PRINT_L0("Number of blocks imported: " << num_imported) + if (h > 0) + // TODO: if there was an error, the last added block is probably at zero-based height h-2 + LOG_PRINT_L0("Finished at block: " << h-1 << " total blocks: " << h); + } + std::cout << ENDL; + return 0; +} + +int main(int argc, char* argv[]) +{ + std::string import_filename = BLOCKCHAIN_RAW; +#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) + std::string default_db_engine = "memory"; +#else + std::string default_db_engine = "lmdb"; +#endif + + uint32_t log_level = LOG_LEVEL_0; + uint64_t num_blocks = 0; + uint64_t block_height = 0; + std::string dirname; + std::string db_arg_str; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor<uint64_t> arg_block_height = {"block-number", "stop at block number", block_height}; + const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor<uint64_t> arg_pop_blocks = {"pop-blocks", "", num_blocks}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + const command_line::arg_descriptor<bool> arg_count_blocks = { + "count-blocks" + , "Count blocks in bootstrap file and exit" + , false + }; + const command_line::arg_descriptor<std::string> arg_database = { + "database", "available: memory, lmdb" + , default_db_engine + }; + const command_line::arg_descriptor<bool> arg_verify = {"verify", + "Verify blocks and transactions during import", true}; + const command_line::arg_descriptor<bool> arg_batch = {"batch", + "Batch transactions for faster import", true}; + const command_line::arg_descriptor<bool> arg_resume = {"resume", + "Resume from current height if output database already exists", true}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_block_height); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_pop_blocks); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_database); + + command_line::add_arg(desc_cmd_only, arg_count_blocks); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + // call add_options() directly for these arguments since + // command_line helpers support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + log_level = command_line::get_arg(vm, arg_log_level); + opt_verify = command_line::get_arg(vm, arg_verify); + opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); + block_height = command_line::get_arg(vm, arg_block_height); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + exit(1); + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + exit(1); + } + if (opt_verify && vm["batch-size"].defaulted()) + { + // usually want batch size default lower if verify on, so progress can be + // frequently saved. + // + // currently, with Windows, default batch size is low, so ignore + // default db_batch_size_verify unless it's even lower + if (db_batch_size > db_batch_size_verify) + { + db_batch_size = db_batch_size_verify; + } + } + uint64_t stop_height = 0; + if (! vm["block-number"].defaulted()) + { + stop_height = block_height; + } + + std::vector<std::string> db_engines {"memory", "lmdb"}; + + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + dirname = command_line::get_arg(vm, data_dir_arg); + db_arg_str = command_line::get_arg(vm, arg_database); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + boost::filesystem::path file_path {dirname}; + std::string import_file_path; + + import_file_path = (file_path / "export" / import_filename).string(); + if (command_line::has_arg(vm, arg_count_blocks)) + { + BootstrapFile bootstrap; + bootstrap.count_blocks(import_file_path); + exit(0); + } + + + std::string db_engine; + int mdb_flags = 0; + int res = 0; + res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + if (res) + { + std::cerr << "Error parsing database argument(s)" << ENDL; + exit(1); + } + + if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) + { + std::cerr << "Invalid database engine: " << db_engine << std::endl; + exit(1); + } + + LOG_PRINT_L0("database: " << db_engine); + LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + LOG_PRINT_L0("bootstrap file path: " << import_file_path); + LOG_PRINT_L0("database path: " << file_path.string()); + + try + { + + // fake_core needed for verification to work when enabled. + // + // NOTE: don't need fake_core method of doing things when we're going to call + // BlockchainDB add_block() directly and have available the 3 block + // properties to do so. Both ways work, but fake core isn't necessary in that + // circumstance. + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + if (db_engine == "lmdb") + { + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); + import_from_file(simple_core, import_file_path, stop_height); + } + else if (db_engine == "memory") + { + fake_core_memory simple_core(dirname, opt_testnet); + import_from_file(simple_core, import_file_path, stop_height); + } + else + { + std::cerr << "database engine unrecognized" << ENDL; + exit(1); + } + + // for multi_db_compile: +#else + if (db_engine != default_db_engine) + { + std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; + exit(1); + } +#if BLOCKCHAIN_DB == DB_LMDB + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); +#else + fake_core_memory simple_core(dirname, opt_testnet); +#endif + + if (! vm["pop-blocks"].defaulted()) + { + num_blocks = command_line::get_arg(vm, arg_pop_blocks); + LOG_PRINT_L0("height: " << simple_core.m_storage.get_current_blockchain_height()); + pop_blocks(simple_core, num_blocks); + LOG_PRINT_L0("height: " << simple_core.m_storage.get_current_blockchain_height()); + exit(0); + } + + import_from_file(simple_core, import_file_path, stop_height); +#endif + + } + catch (const DB_ERROR& e) + { + std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL; + exit(1); + } + + // destructors called at exit: + // + // ensure db closed + // - transactions properly checked and handled + // - disk sync if needed + // + // fake_core object's destructor is called when it goes out of scope. For an + // LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn + // calls delete on its BlockchainDB derived class' object, which closes its + // files. +} diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp new file mode 100644 index 000000000..fb67e12bc --- /dev/null +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -0,0 +1,501 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "bootstrap_serialization.h" +#include "serialization/binary_utils.h" // dump_binary(), parse_binary() +#include "serialization/json_utils.h" // dump_json() + +#include "bootstrap_file.h" + + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +namespace +{ + // This number was picked by taking the leading 4 bytes from this output: + // echo Monero bootstrap file | sha1sum + const uint32_t blockchain_raw_magic = 0x28721586; + const uint32_t header_size = 1024; + + std::string refresh_string = "\r \r"; +} + + + +bool BootstrapFile::open_writer(const boost::filesystem::path& dir_path) +{ + if (boost::filesystem::exists(dir_path)) + { + if (!boost::filesystem::is_directory(dir_path)) + { + LOG_PRINT_RED_L0("export directory path is a file: " << dir_path); + return false; + } + } + else + { + if (!boost::filesystem::create_directory(dir_path)) + { + LOG_PRINT_RED_L0("Failed to create directory " << dir_path); + return false; + } + } + + std::string file_path = (dir_path / BLOCKCHAIN_RAW).string(); + m_raw_data_file = new std::ofstream(); + + bool do_initialize_file = false; + uint64_t num_blocks = 0; + + if (! boost::filesystem::exists(file_path)) + { + LOG_PRINT_L0("creating file"); + do_initialize_file = true; + num_blocks = 0; + } + else + { + num_blocks = count_blocks(file_path); + LOG_PRINT_L0("appending to existing file with height: " << num_blocks-1 << " total blocks: " << num_blocks); + } + m_height = num_blocks; + + if (do_initialize_file) + m_raw_data_file->open(file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + else + m_raw_data_file->open(file_path, std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate); + + if (m_raw_data_file->fail()) + return false; + + m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer); + if (m_output_stream == nullptr) + return false; + + if (do_initialize_file) + initialize_file(); + + return true; +} + + +bool BootstrapFile::initialize_file() +{ + const uint32_t file_magic = blockchain_raw_magic; + + std::string blob; + if (! ::serialization::dump_binary(file_magic, blob)) + { + throw std::runtime_error("Error in serialization of file magic"); + } + *m_raw_data_file << blob; + + bootstrap::file_info bfi; + bfi.major_version = 0; + bfi.minor_version = 1; + bfi.header_size = header_size; + + bootstrap::blocks_info bbi; + bbi.block_first = 0; + bbi.block_last = 0; + bbi.block_last_pos = 0; + + buffer_type buffer2; + boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* output_stream_header; + output_stream_header = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(buffer2); + + uint32_t bd_size = 0; + + blobdata bd = t_serializable_object_to_blob(bfi); + LOG_PRINT_L1("bootstrap::file_info size: " << bd.size()); + bd_size = bd.size(); + + if (! ::serialization::dump_binary(bd_size, blob)) + { + throw std::runtime_error("Error in serialization of bootstrap::file_info size"); + } + *output_stream_header << blob; + *output_stream_header << bd; + + bd = t_serializable_object_to_blob(bbi); + LOG_PRINT_L1("bootstrap::blocks_info size: " << bd.size()); + bd_size = bd.size(); + + if (! ::serialization::dump_binary(bd_size, blob)) + { + throw std::runtime_error("Error in serialization of bootstrap::blocks_info size"); + } + *output_stream_header << blob; + *output_stream_header << bd; + + output_stream_header->flush(); + *output_stream_header << std::string(header_size-buffer2.size(), 0); // fill in rest with null bytes + output_stream_header->flush(); + std::copy(buffer2.begin(), buffer2.end(), std::ostreambuf_iterator<char>(*m_raw_data_file)); + + return true; +} + +void BootstrapFile::flush_chunk() +{ + m_output_stream->flush(); + + uint32_t chunk_size = m_buffer.size(); + // LOG_PRINT_L0("chunk_size " << chunk_size); + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + } + + std::string blob; + if (! ::serialization::dump_binary(chunk_size, blob)) + { + throw std::runtime_error("Error in serialization of chunk size"); + } + *m_raw_data_file << blob; + + if (m_max_chunk < chunk_size) + { + m_max_chunk = chunk_size; + } + long pos_before = m_raw_data_file->tellp(); + std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator<char>(*m_raw_data_file)); + m_raw_data_file->flush(); + long pos_after = m_raw_data_file->tellp(); + long num_chars_written = pos_after - pos_before; + if (static_cast<unsigned long>(num_chars_written) != chunk_size) + { + LOG_PRINT_RED_L0("Error writing chunk: height: " << m_cur_height << " chunk_size: " << chunk_size << " num chars written: " << num_chars_written); + throw std::runtime_error("Error writing chunk"); + } + + m_buffer.clear(); + delete m_output_stream; + m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer); + LOG_PRINT_L1("flushed chunk: chunk_size: " << chunk_size); +} + +void BootstrapFile::write_block(block& block) +{ + bootstrap::block_package bp; + bp.block = block; + + std::vector<transaction> txs; + + uint64_t block_height = boost::get<txin_gen>(block.miner_tx.vin.front()).height; + + + // now add all regular transactions + for (const auto& tx_id : block.tx_hashes) + { + if (tx_id == null_hash) + { + throw std::runtime_error("Aborting: tx == null_hash"); + } +#if SOURCE_DB == DB_MEMORY + const transaction* tx = m_blockchain_storage->get_tx(tx_id); +#else + transaction tx = m_blockchain_storage->get_db().get_tx(tx_id); +#endif + +#if SOURCE_DB == DB_MEMORY + if(tx == NULL) + { + if (! m_tx_pool) + throw std::runtime_error("Aborting: tx == NULL, so memory pool required to get tx, but memory pool isn't enabled"); + else + { + transaction tx; + if(m_tx_pool->get_transaction(tx_id, tx)) + txs.push_back(tx); + else + throw std::runtime_error("Aborting: tx not found in pool"); + } + } + else + txs.push_back(*tx); +#else + txs.push_back(tx); +#endif + } + + // these non-coinbase txs will be serialized using this structure + bp.txs = txs; + + // These three attributes are currently necessary for a fast import that adds blocks without verification. + bool include_extra_block_data = true; + if (include_extra_block_data) + { +#if SOURCE_DB == DB_MEMORY + size_t block_size = m_blockchain_storage->get_block_size(block_height); + difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height); + uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height); +#else + size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); + difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); + uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); +#endif + + bp.block_size = block_size; + bp.cumulative_difficulty = cumulative_difficulty; + bp.coins_generated = coins_generated; + } + + blobdata bd = t_serializable_object_to_blob(bp); + m_output_stream->write((const char*)bd.data(), bd.size()); +} + +bool BootstrapFile::close() +{ + if (m_raw_data_file->fail()) + return false; + + m_raw_data_file->flush(); + delete m_output_stream; + delete m_raw_data_file; + return true; +} + + +#if SOURCE_DB == DB_MEMORY +bool BootstrapFile::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) +#else +bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) +#endif +{ + uint64_t num_blocks_written = 0; + m_max_chunk = 0; + m_blockchain_storage = _blockchain_storage; + m_tx_pool = _tx_pool; + uint64_t progress_interval = 100; + LOG_PRINT_L0("Storing blocks raw data..."); + if (!BootstrapFile::open_writer(output_dir)) + { + LOG_PRINT_RED_L0("failed to open raw file for write"); + return false; + } + block b; + uint64_t height_start = m_height; // height_start uses 0-based height, m_height uses 1-based height. so height_start doesn't need to add 1 here, as it's already at the next height + uint64_t height_stop = 0; + LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1); + if ((requested_block_height > 0) && (requested_block_height < m_blockchain_storage->get_current_blockchain_height())) + { + LOG_PRINT_L0("Using requested block height: " << requested_block_height); + height_stop = requested_block_height; + } + else + { + height_stop = m_blockchain_storage->get_current_blockchain_height() - 1; + LOG_PRINT_L0("Using block height of source blockchain: " << height_stop); + } + for (m_cur_height = height_start; m_cur_height <= height_stop; ++m_cur_height) + { + // this method's height refers to 0-based height (genesis block = height 0) + crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height); + m_blockchain_storage->get_block_by_hash(hash, b); + write_block(b); + if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) { + flush_chunk(); + num_blocks_written += NUM_BLOCKS_PER_CHUNK; + } + if (m_cur_height % progress_interval == 0) { + std::cout << refresh_string; + std::cout << "block " << m_cur_height << "/" << height_stop << std::flush; + } + } + // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported. + if (m_cur_height % NUM_BLOCKS_PER_CHUNK != 0) + { + flush_chunk(); + } + // print message for last block, which may not have been printed yet due to progress_interval + std::cout << refresh_string; + std::cout << "block " << m_cur_height-1 << "/" << height_stop << ENDL; + + LOG_PRINT_L0("Number of blocks exported: " << num_blocks_written); + if (num_blocks_written > 0) + LOG_PRINT_L0("Largest chunk: " << m_max_chunk << " bytes"); + + return BootstrapFile::close(); +} + +uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file) +{ + uint32_t file_magic; + + std::string str1; + char buf1[2048]; + import_file.read(buf1, sizeof(file_magic)); + if (! import_file) + throw std::runtime_error("Error reading expected number of bytes"); + str1.assign(buf1, sizeof(file_magic)); + + if (! ::serialization::parse_binary(str1, file_magic)) + throw std::runtime_error("Error in deserialization of file_magic"); + + if (file_magic != blockchain_raw_magic) + { + LOG_PRINT_RED_L0("bootstrap file not recognized"); + throw std::runtime_error("Aborting"); + } + else + LOG_PRINT_L0("bootstrap file recognized"); + + uint32_t buflen_file_info; + + import_file.read(buf1, sizeof(buflen_file_info)); + str1.assign(buf1, sizeof(buflen_file_info)); + if (! import_file) + throw std::runtime_error("Error reading expected number of bytes"); + if (! ::serialization::parse_binary(str1, buflen_file_info)) + throw std::runtime_error("Error in deserialization of buflen_file_info"); + LOG_PRINT_L1("bootstrap::file_info size: " << buflen_file_info); + + if (buflen_file_info > sizeof(buf1)) + throw std::runtime_error("Error: bootstrap::file_info size exceeds buffer size"); + import_file.read(buf1, buflen_file_info); + if (! import_file) + throw std::runtime_error("Error reading expected number of bytes"); + str1.assign(buf1, buflen_file_info); + bootstrap::file_info bfi; + if (! ::serialization::parse_binary(str1, bfi)) + throw std::runtime_error("Error in deserialization of bootstrap::file_info"); + LOG_PRINT_L0("bootstrap file v" << unsigned(bfi.major_version) << "." << unsigned(bfi.minor_version)); + LOG_PRINT_L0("bootstrap magic size: " << sizeof(file_magic)); + LOG_PRINT_L0("bootstrap header size: " << bfi.header_size) + + uint64_t full_header_size = sizeof(file_magic) + bfi.header_size; + import_file.seekg(full_header_size); + + return full_header_size; +} + +uint64_t BootstrapFile::count_blocks(const std::string& import_file_path) +{ + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("bootstrap file not found: " << raw_file_path); + throw std::runtime_error("Aborting"); + } + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + throw std::runtime_error("Aborting"); + } + + uint64_t full_header_size; // 4 byte magic + length of header structures + full_header_size = seek_to_first_chunk(import_file); + + LOG_PRINT_L0("Scanning blockchain from bootstrap file..."); + block b; + bool quit = false; + uint64_t bytes_read = 0; + int progress_interval = 10; + + std::string str1; + char buf1[2048]; + while (! quit) + { + uint32_t chunk_size; + import_file.read(buf1, sizeof(chunk_size)); + if (!import_file) { + std::cout << refresh_string; + LOG_PRINT_L1("End of file reached"); + quit = true; + break; + } + h += NUM_BLOCKS_PER_CHUNK; + if ((h-1) % progress_interval == 0) + { + std::cout << "\r" << "block height: " << h-1 << + " " << + std::flush; + } + bytes_read += sizeof(chunk_size); + + str1.assign(buf1, sizeof(chunk_size)); + if (! ::serialization::parse_binary(str1, chunk_size)) + throw std::runtime_error("Error in deserialization of chunk_size"); + LOG_PRINT_L1("chunk_size: " << chunk_size); + + if (chunk_size > BUFFER_SIZE) + { + std::cout << refresh_string; + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE + << " height: " << h-1); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + std::cout << refresh_string; + LOG_PRINT_L0("NOTE: chunk_size " << chunk_size << " > 100000" << " height: " + << h-1); + } + else if (chunk_size <= 0) { + std::cout << refresh_string; + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1); + throw std::runtime_error("Aborting"); + } + // skip to next expected block size value + import_file.seekg(chunk_size, std::ios_base::cur); + if (! import_file) { + std::cout << refresh_string; + LOG_PRINT_L0("ERROR: unexpected end of file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + throw std::runtime_error("Aborting"); + } + bytes_read += chunk_size; + + // std::cout << refresh_string; + LOG_PRINT_L3("Number bytes scanned: " << bytes_read); + } + + import_file.close(); + + std::cout << ENDL; + std::cout << "Done scanning bootstrap file" << ENDL; + std::cout << "Full header length: " << full_header_size << " bytes" << ENDL; + std::cout << "Scanned for blocks: " << bytes_read << " bytes" << ENDL; + std::cout << "Total: " << full_header_size + bytes_read << " bytes" << ENDL; + std::cout << "Number of blocks: " << h << ENDL; + std::cout << ENDL; + + // NOTE: h is the number of blocks. + // Note that a block's stored height is zero-based, but parts of the code use + // one-based height. + return h; +} diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h new file mode 100644 index 000000000..5fb8a1d4a --- /dev/null +++ b/src/blockchain_utilities/bootstrap_file.h @@ -0,0 +1,116 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/iostreams/stream_buffer.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/iostreams/device/back_inserter.hpp> + +#include <boost/iostreams/filtering_streambuf.hpp> + +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +#include <algorithm> +#include <cstdio> +#include <fstream> +#include <boost/iostreams/copy.hpp> +#include <atomic> + +#include "common/command_line.h" +#include "version.h" + + +// CONFIG: choose one of the three #define's +// +// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows +// the exporter to use the in-memory blockchain while the other binaries +// work with LMDB, without recompiling anything. +// +#define SOURCE_DB DB_MEMORY +// #define SOURCE_DB DB_LMDB +// to use global compile-time setting (DB_MEMORY or DB_LMDB): +// #define SOURCE_DB BLOCKCHAIN_DB + + +// bounds checking is done before writing to buffer, but buffer size +// should be a sensible maximum +#define BUFFER_SIZE 1000000 +#define NUM_BLOCKS_PER_CHUNK 1 +#define BLOCKCHAIN_RAW "blockchain.raw" + + +using namespace cryptonote; + + +class BootstrapFile +{ +public: + + uint64_t count_blocks(const std::string& dir_path); + uint64_t seek_to_first_chunk(std::ifstream& import_file); + +#if SOURCE_DB == DB_MEMORY + bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#else + bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#endif + +protected: + +#if SOURCE_DB == DB_MEMORY + blockchain_storage* m_blockchain_storage; +#else + Blockchain* m_blockchain_storage; +#endif + + tx_memory_pool* m_tx_pool; + typedef std::vector<char> buffer_type; + std::ofstream * m_raw_data_file; + buffer_type m_buffer; + boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream; + + // open export file for write + bool open_writer(const boost::filesystem::path& dir_path); + bool initialize_file(); + bool close(); + void write_block(block& block); + void flush_chunk(); + +private: + + uint64_t m_height; + uint64_t m_cur_height; // tracks current height during export + uint32_t m_max_chunk; +}; diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h new file mode 100644 index 000000000..6fa949353 --- /dev/null +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -0,0 +1,88 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cryptonote_core/cryptonote_boost_serialization.h" +#include "cryptonote_core/difficulty.h" + + +namespace cryptonote +{ + namespace bootstrap + { + + struct file_info + { + uint8_t major_version; + uint8_t minor_version; + uint32_t header_size; + + BEGIN_SERIALIZE_OBJECT() + FIELD(major_version); + FIELD(minor_version); + VARINT_FIELD(header_size); + END_SERIALIZE() + }; + + struct blocks_info + { + // block heights of file's first and last blocks, zero-based indexes + uint64_t block_first; + uint64_t block_last; + + // file position, for directly reading last block + uint64_t block_last_pos; + + BEGIN_SERIALIZE_OBJECT() + VARINT_FIELD(block_first); + VARINT_FIELD(block_last); + VARINT_FIELD(block_last_pos); + END_SERIALIZE() + }; + + struct block_package + { + cryptonote::block block; + std::vector<transaction> txs; + size_t block_size; + difficulty_type cumulative_difficulty; + uint64_t coins_generated; + + BEGIN_SERIALIZE() + FIELD(block) + FIELD(txs) + VARINT_FIELD(block_size) + VARINT_FIELD(cumulative_difficulty) + VARINT_FIELD(coins_generated) + END_SERIALIZE() + }; + + } + +} diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h new file mode 100644 index 000000000..79fb51842 --- /dev/null +++ b/src/blockchain_utilities/fake_core.h @@ -0,0 +1,165 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/filesystem.hpp> +#include "cryptonote_core/blockchain.h" // BlockchainDB +#include "cryptonote_core/blockchain_storage.h" // in-memory DB +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +using namespace cryptonote; + + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB + +struct fake_core_lmdb +{ + Blockchain m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) + // for multi_db_compile: +#else + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) +#endif + { + m_pool.init(path.string()); + + BlockchainDB* db = new BlockchainLMDB(); + + boost::filesystem::path folder(path); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.string() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename, mdb_flags); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + throw; + } + + m_storage.init(db, use_testnet); + if (do_batch) + m_storage.get_db().set_batch_transactions(do_batch); + support_batch = true; + support_add_block = true; + } + ~fake_core_lmdb() + { + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) + { + return m_storage.get_db().add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + } + + void batch_start() + { + m_storage.get_db().batch_start(); + } + + void batch_stop() + { + m_storage.get_db().batch_stop(); + } + +}; +#endif + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_MEMORY + +struct fake_core_memory +{ + blockchain_storage m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(&m_storage), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(m_storage), m_storage(&m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + support_batch = false; + support_add_block = false; + } + ~fake_core_memory() + { + LOG_PRINT_L3("fake_core_memory() destructor called - want to see it ripple down"); + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) + { + // TODO: + // would need to refactor handle_block_to_main_chain() to have a direct add_block() method like Blockchain class + throw std::runtime_error("direct add_block() method not implemented for in-memory db"); + return 2; + } + + void batch_start() + { + LOG_PRINT_L0("WARNING: [batch_start] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + + void batch_stop() + { + LOG_PRINT_L0("WARNING: [batch_stop] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + +}; + +#endif |