From 4af0918501c40c20f9b42d1d2d7da154d365a58b Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 1 Nov 2014 20:55:22 -0400 Subject: very, VERY primitive blockchain converter hard-coded config folder, hard-coded BlockchainDB subclass. Needs finessing, but should be testable this way. update for rebase (warptangent 2015-01-04) fix conflicts with upstream CMakeLists.txt files src/CMakeLists.txt (edit original commit) src/blockchain_converter/CMakeLists.txt (add) --- src/blockchain_converter/blockchain_converter.cpp | 99 +++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 src/blockchain_converter/blockchain_converter.cpp (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp new file mode 100644 index 000000000..5dc937b3b --- /dev/null +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -0,0 +1,99 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "common/util.h" +#include "warnings.h" +#include "crypto/crypto.h" +#include "cryptonote_config.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "misc_language.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain_db.h" +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" +#include "cryptonote_core/tx_pool.h" + +using namespace cryptonote; + +struct fake_core +{ + tx_memory_pool m_pool; + Blockchain dummy; + + blockchain_storage m_storage; + + + fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) + { + m_pool.init("~/.bitmonero"); + m_storage.init("~/.bitmonero", false); + } +}; + +int main(int argc, char* argv[]) +{ + fake_core c; + + BlockchainDB *blockchain; + + blockchain = new BlockchainLMDB(); + + blockchain->open("~/.bitmonero"); + + for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) + { + block b = c.m_storage.get_block(i); + size_t bsize = c.m_storage.get_block_size(i); + difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); + uint64_t bcoins = c.m_storage.get_block_coins_generated(i); + std::vector txs; + std::vector missed; + + c.m_storage.get_transactions(b.tx_hashes, txs, missed); + if (missed.size()) + { + std::cerr << "Missed transaction(s) for block at height " << i << ", exiting" << std::endl; + delete blockchain; + return 1; + } + + try + { + blockchain->add_block(b, bsize, bdiff, bcoins, txs); + } + catch (const std::exception& e) + { + std::cerr << "Error adding block to new blockchain: " << e.what() << std::endl; + delete blockchain; + return 2; + } + } + + return 0; +} -- cgit v1.2.3 From 9455e0cd58871812294ceeeef44b053aea8e0671 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 1 Nov 2014 21:13:15 -0400 Subject: ~ didn't work, need hard path. debug print. --- src/blockchain_converter/blockchain_converter.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 5dc937b3b..b18fd3525 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -38,6 +38,7 @@ #include "cryptonote_core/blockchain.h" #include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" #include "cryptonote_core/tx_pool.h" +#include using namespace cryptonote; @@ -51,8 +52,8 @@ struct fake_core fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) { - m_pool.init("~/.bitmonero"); - m_storage.init("~/.bitmonero", false); + m_pool.init("/home/user/.bitmonero"); + m_storage.init("/home/user/.bitmonero", false); } }; @@ -64,10 +65,11 @@ int main(int argc, char* argv[]) blockchain = new BlockchainLMDB(); - blockchain->open("~/.bitmonero"); + blockchain->open("/home/user/.bitmonero"); for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) { + if (i % 10 == 0) std::cout << "block " << i << std::endl; block b = c.m_storage.get_block(i); size_t bsize = c.m_storage.get_block_size(i); difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); -- cgit v1.2.3 From 11129b9ee4e1f022ca1ae1c26d8f50e8aa8d3e03 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 16:12:25 +0000 Subject: blockchain_converter: use the actual blockchain location --- src/blockchain_converter/blockchain_converter.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index b18fd3525..a2b3a375f 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -52,20 +52,22 @@ struct fake_core fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) { - m_pool.init("/home/user/.bitmonero"); - m_storage.init("/home/user/.bitmonero", false); + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + m_pool.init(default_data_path.string()); + m_storage.init(default_data_path.string(), false); } }; int main(int argc, char* argv[]) { fake_core c; + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; BlockchainDB *blockchain; blockchain = new BlockchainLMDB(); - blockchain->open("/home/user/.bitmonero"); + blockchain->open(default_data_path.string()); for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) { -- cgit v1.2.3 From 98bdadcad7a276f2196cfebfe2148d36f65872d1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 22:53:27 +0000 Subject: blockchain_converter: delete blockchain on succesful exit While the dtor implementation does not actually do anything, other paths do delete it, and the dtor might do someting later. --- src/blockchain_converter/blockchain_converter.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index a2b3a375f..4653cf66c 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -99,5 +99,6 @@ int main(int argc, char* argv[]) } } + delete blockchain; return 0; } -- cgit v1.2.3 From 2b9f73787259dfbf164516925beb5befd8f1df6b Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 10:28:46 +0000 Subject: blockchain_converter: only call data path function once --- src/blockchain_converter/blockchain_converter.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 4653cf66c..1ae1efc72 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -50,18 +50,17 @@ struct fake_core blockchain_storage m_storage; - fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) + fake_core(const boost::filesystem::path &path) : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) { - boost::filesystem::path default_data_path {tools::get_default_data_dir()}; - m_pool.init(default_data_path.string()); - m_storage.init(default_data_path.string(), false); + m_pool.init(path.string()); + m_storage.init(path.string(), false); } }; int main(int argc, char* argv[]) { - fake_core c; boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + fake_core c(default_data_path); BlockchainDB *blockchain; -- cgit v1.2.3 From 609cf7fc92987da0585651a6e95225575ecacd09 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 11:01:17 +0000 Subject: blockchain_converter: a bit more user friendly output --- src/blockchain_converter/blockchain_converter.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 1ae1efc72..f2dd10b18 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -68,9 +68,13 @@ int main(int argc, char* argv[]) blockchain->open(default_data_path.string()); - for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) + for (uint64_t height, i = 0; i < (height = c.m_storage.get_current_blockchain_height()); ++i) { - if (i % 10 == 0) std::cout << "block " << i << std::endl; + if (i % 10 == 0) + { + std::cout << "\r \r" << "block " << i << "/" << height + << " (" << (i+1)*100/height<< "%)" << std::flush; + } block b = c.m_storage.get_block(i); size_t bsize = c.m_storage.get_block_size(i); difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); @@ -81,6 +85,7 @@ int main(int argc, char* argv[]) c.m_storage.get_transactions(b.tx_hashes, txs, missed); if (missed.size()) { + std::cout << std::endl; std::cerr << "Missed transaction(s) for block at height " << i << ", exiting" << std::endl; delete blockchain; return 1; @@ -92,12 +97,14 @@ int main(int argc, char* argv[]) } catch (const std::exception& e) { + std::cout << std::endl; std::cerr << "Error adding block to new blockchain: " << e.what() << std::endl; delete blockchain; return 2; } } + std::cout << std::endl; delete blockchain; return 0; } -- cgit v1.2.3 From 1362846dd7c7e50b8804e24ece1a750472cc4485 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 19 Dec 2014 18:18:21 +0000 Subject: blockchain_converter: add --testnet for converting testnet blockchain --- src/blockchain_converter/blockchain_converter.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index f2dd10b18..57822700f 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -59,7 +59,12 @@ struct fake_core int main(int argc, char* argv[]) { - boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + std::string dir = tools::get_default_data_dir(); + boost::filesystem::path default_data_path {dir}; + if (argc >= 2 && !strcmp(argv[1], "--testnet")) { + default_data_path /= "testnet"; + } + fake_core c(default_data_path); BlockchainDB *blockchain; -- cgit v1.2.3 From cd972bdcc25e9ed1d4c5bc7663242afa6a61125e Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 19:54:27 -0800 Subject: Update year and formatting in license --- src/blockchain_converter/blockchain_converter.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 57822700f..0dfc1a2c3 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014, The Monero Project -// +// Copyright (c) 2014-2015, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -- cgit v1.2.3 From 5eab480cb116b7c58fe020951995815baa7ccd8f Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 6 Mar 2015 15:20:45 -0500 Subject: Moved BlockchainDB into its own src/ subfolder Ostensibly janitorial work, but should be more relevant later down the line. Things that depend on core cryptonote things (i.e. cryptonote_core) don't necessarily depend on BlockchainDB and thus have no need to have BlockchainDB baked in with them. --- src/blockchain_converter/blockchain_converter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 0dfc1a2c3..aae569e58 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -34,9 +34,9 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "misc_language.h" #include "cryptonote_core/blockchain_storage.h" -#include "cryptonote_core/blockchain_db.h" +#include "blockchain_db/blockchain_db.h" #include "cryptonote_core/blockchain.h" -#include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" +#include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_core/tx_pool.h" #include -- cgit v1.2.3 From eee3ee70730bc3bf7874e5f22139a55dac3a2cdd Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 13 Mar 2015 21:39:27 -0400 Subject: BlockchainDB implementations have names now In order to make things more general, BlockchainDB now has get_db_name() which should return a string with the "name" of that type of db. This "name" will be the subfolder name that holds that db type's files within the monero folder. Small bugfix: blockchain_converter was not correctly appending this in the prior hard-coded-string implementation of the subfolder data directory concept. --- src/blockchain_converter/blockchain_converter.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index aae569e58..8ac9b81bb 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -71,7 +71,11 @@ int main(int argc, char* argv[]) blockchain = new BlockchainLMDB(); - blockchain->open(default_data_path.string()); + boost::filesystem::path db_path(default_data_path); + + db_path /= blockchain->get_db_name(); + + blockchain->open(db_path.string()); for (uint64_t height, i = 0; i < (height = c.m_storage.get_current_blockchain_height()); ++i) { -- cgit v1.2.3 From a3dd9d10f39bddfb386370eb01e76cbbb0fe6fbb Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 3 Mar 2015 17:20:27 -0800 Subject: blockchain_converter: Add support for batch transactions Add log level support. Add testnet support. Add command-line options: --help --data-dir --testnet-data-dir --testnet --log-level --batch --batch-size --block-number See help for usage. Run at log level 1 to see profiling stats. --- src/blockchain_converter/blockchain_converter.cpp | 196 +++++++++++++++++++--- 1 file changed, 173 insertions(+), 23 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 8ac9b81bb..eaf799a36 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -38,52 +38,179 @@ #include "cryptonote_core/blockchain.h" #include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_core/tx_pool.h" +#include "common/command_line.h" +#include "serialization/json_utils.h" +#include "include_base_utils.h" +#include "version.h" #include +// CONFIG +static bool opt_batch = true; +static bool opt_testnet = false; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +static uint64_t db_batch_size = 20000; + + +namespace po = boost::program_options; + using namespace cryptonote; +using namespace epee; struct fake_core { - tx_memory_pool m_pool; Blockchain dummy; + tx_memory_pool m_pool; blockchain_storage m_storage; - - fake_core(const boost::filesystem::path &path) : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) +#if !defined(BLOCKCHAIN_DB) + // for multi_db_runtime: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(&dummy), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(dummy), m_storage(&m_pool) +#endif { m_pool.init(path.string()); - m_storage.init(path.string(), false); + m_storage.init(path.string(), use_testnet); } }; int main(int argc, char* argv[]) { - std::string dir = tools::get_default_data_dir(); - boost::filesystem::path default_data_path {dir}; - if (argc >= 2 && !strcmp(argv[1], "--testnet")) { - default_data_path /= "testnet"; + uint64_t height = 0; + uint64_t start_block = 0; + uint64_t end_block = 0; + uint64_t num_blocks = 0; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "", LOG_LEVEL_0}; + const command_line::arg_descriptor arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor arg_testnet_on = { + "testnet" + , "Run on testnet." + , opt_testnet + }; + const command_line::arg_descriptor arg_block_number = + {"block-number", "Number of blocks (default: use entire source blockchain)", + 0}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_block_number); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + const command_line::arg_descriptor arg_batch = {"batch", + "Batch transactions for faster import", true}; + + // call add_options() directly for these arguments since command_line helpers + // support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + + return true; + }); + if (!r) + return 1; + + int log_level = command_line::get_arg(vm, arg_log_level); + opt_batch = command_line::get_arg(vm, arg_batch); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; } - fake_core c(default_data_path); + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + return 1; + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + return 1; + } - BlockchainDB *blockchain; + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); - blockchain = new BlockchainLMDB(); + std::string src_folder; + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + src_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path dest_folder(src_folder); - boost::filesystem::path db_path(default_data_path); + num_blocks = command_line::get_arg(vm, arg_block_number); - db_path /= blockchain->get_db_name(); + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + fake_core c(src_folder, opt_testnet); - blockchain->open(db_path.string()); + height = c.m_storage.get_current_blockchain_height(); + if (! num_blocks || num_blocks > height) + end_block = height - 1; + else + end_block = start_block + num_blocks - 1; + + BlockchainDB *blockchain; + blockchain = new BlockchainLMDB(opt_batch); + dest_folder /= blockchain->get_db_name(); + LOG_PRINT_L0("Source blockchain: " << src_folder); + LOG_PRINT_L0("Dest blockchain: " << dest_folder.string()); + LOG_PRINT_L0("Opening LMDB: " << dest_folder.string()); + blockchain->open(dest_folder.string()); - for (uint64_t height, i = 0; i < (height = c.m_storage.get_current_blockchain_height()); ++i) + if (opt_batch) + blockchain->batch_start(); + uint64_t i = 0; + for (i = start_block; i < end_block + 1; ++i) { - if (i % 10 == 0) + // block: i height: i+1 end height: end_block + 1 + if ((i+1) % 10 == 0) { - std::cout << "\r \r" << "block " << i << "/" << height - << " (" << (i+1)*100/height<< "%)" << std::flush; + std::cout << "\r \r" << "height " << i+1 << "/" << + end_block+1 << " (" << (i+1)*100/(end_block+1)<< "%)" << std::flush; } + // for debugging: + // std::cout << "height " << i+1 << "/" << end_block+1 + // << " ((" << i+1 << ")*100/(end_block+1))" << "%)" << ENDL; + block b = c.m_storage.get_block(i); size_t bsize = c.m_storage.get_block_size(i); difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); @@ -94,8 +221,8 @@ int main(int argc, char* argv[]) c.m_storage.get_transactions(b.tx_hashes, txs, missed); if (missed.size()) { - std::cout << std::endl; - std::cerr << "Missed transaction(s) for block at height " << i << ", exiting" << std::endl; + std::cout << ENDL; + std::cerr << "Missed transaction(s) for block at height " << i + 1 << ", exiting" << ENDL; delete blockchain; return 1; } @@ -103,17 +230,40 @@ int main(int argc, char* argv[]) try { blockchain->add_block(b, bsize, bdiff, bcoins, txs); + + if (opt_batch) + { + if ((i < end_block) && ((i + 1) % db_batch_size == 0)) + { + std::cout << "\r \r"; + std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL; + blockchain->batch_stop(); + blockchain->batch_start(); + std::cout << ENDL; + blockchain->show_stats(); + } + } } catch (const std::exception& e) { - std::cout << std::endl; - std::cerr << "Error adding block to new blockchain: " << e.what() << std::endl; + std::cout << ENDL; + std::cerr << "Error adding block to new blockchain: " << e.what() << ENDL; delete blockchain; return 2; } } + if (opt_batch) + { + std::cout << "\r \r" << "height " << i << "/" << + end_block+1 << " (" << (i)*100/(end_block+1)<< "%)" << std::flush; + std::cout << ENDL; + std::cout << "[- batch commit at height " << i << " -]" << ENDL; + blockchain->batch_stop(); + } + std::cout << ENDL; + blockchain->show_stats(); + std::cout << "Finished at height: " << i << " block: " << i-1 << ENDL; - std::cout << std::endl; delete blockchain; return 0; } -- cgit v1.2.3 From dbdcf11778ac9500ce55affae8db89ac868c42da Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Mar 2015 12:33:50 -0700 Subject: blockchain_converter: Add support for resume from last block Add option "--resume " where default is on. --- src/blockchain_converter/blockchain_converter.cpp | 47 ++++++++++++++++++----- 1 file changed, 37 insertions(+), 10 deletions(-) (limited to 'src/blockchain_converter/blockchain_converter.cpp') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index eaf799a36..03ab5e434 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -44,14 +44,19 @@ #include "version.h" #include +namespace +{ + // CONFIG -static bool opt_batch = true; -static bool opt_testnet = false; +bool opt_batch = true; +bool opt_resume = true; +bool opt_testnet = false; // number of blocks per batch transaction // adjustable through command-line argument according to available RAM -static uint64_t db_batch_size = 20000; +uint64_t db_batch_size = 20000; +} namespace po = boost::program_options; @@ -113,11 +118,14 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor arg_batch = {"batch", "Batch transactions for faster import", true}; + const command_line::arg_descriptor arg_resume = {"resume", + "Resume from current height if output database already exists", true}; // call add_options() directly for these arguments since command_line helpers // support only boolean switch, not boolean argument desc_cmd_sett.add_options() - (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) ; po::options_description desc_options("Allowed options"); @@ -137,6 +145,7 @@ int main(int argc, char* argv[]) int log_level = command_line::get_arg(vm, arg_log_level); opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); db_batch_size = command_line::get_arg(vm, arg_batch_size); if (command_line::get_arg(vm, command_line::arg_help)) @@ -178,23 +187,41 @@ int main(int argc, char* argv[]) { LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); fake_core c(src_folder, opt_testnet); height = c.m_storage.get_current_blockchain_height(); - if (! num_blocks || num_blocks > height) - end_block = height - 1; - else - end_block = start_block + num_blocks - 1; BlockchainDB *blockchain; blockchain = new BlockchainLMDB(opt_batch); dest_folder /= blockchain->get_db_name(); LOG_PRINT_L0("Source blockchain: " << src_folder); LOG_PRINT_L0("Dest blockchain: " << dest_folder.string()); - LOG_PRINT_L0("Opening LMDB: " << dest_folder.string()); + LOG_PRINT_L0("Opening dest blockchain (BlockchainDB " << blockchain->get_db_name() << ")"); blockchain->open(dest_folder.string()); + LOG_PRINT_L0("Source blockchain height: " << height); + LOG_PRINT_L0("Dest blockchain height: " << blockchain->height()); + + if (opt_resume) + // next block number to add is same as current height + start_block = blockchain->height(); + + if (! num_blocks || (start_block + num_blocks > height)) + end_block = height - 1; + else + end_block = start_block + num_blocks - 1; + + LOG_PRINT_L0("start height: " << start_block+1 << " stop height: " << + end_block+1); + + if (start_block > end_block) + { + LOG_PRINT_L0("Finished: no blocks to add"); + delete blockchain; + return 0; + } if (opt_batch) blockchain->batch_start(); @@ -247,7 +274,7 @@ int main(int argc, char* argv[]) catch (const std::exception& e) { std::cout << ENDL; - std::cerr << "Error adding block to new blockchain: " << e.what() << ENDL; + std::cerr << "Error adding block " << i << " to new blockchain: " << e.what() << ENDL; delete blockchain; return 2; } -- cgit v1.2.3