diff options
Diffstat (limited to 'src/blockchain_utilities')
| -rw-r--r-- | src/blockchain_utilities/CMakeLists.txt | 31 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_dump.cpp | 431 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_export.cpp | 17 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_utilities.h | 51 | ||||
| -rw-r--r-- | src/blockchain_utilities/blocksdat_file.cpp | 170 | ||||
| -rw-r--r-- | src/blockchain_utilities/blocksdat_file.h | 90 | ||||
| -rw-r--r-- | src/blockchain_utilities/bootstrap_file.cpp | 23 | ||||
| -rw-r--r-- | src/blockchain_utilities/bootstrap_file.h | 19 |
8 files changed, 802 insertions, 30 deletions
diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 5be37c450..f88180686 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -38,11 +38,13 @@ bitmonero_private_headers(blockchain_converter set(blockchain_import_sources blockchain_import.cpp bootstrap_file.cpp + blocksdat_file.cpp ) set(blockchain_import_private_headers fake_core.h bootstrap_file.h + blocksdat_file.h bootstrap_serialization.h ) @@ -52,10 +54,12 @@ bitmonero_private_headers(blockchain_import set(blockchain_export_sources blockchain_export.cpp bootstrap_file.cpp + blocksdat_file.cpp ) set(blockchain_export_private_headers bootstrap_file.h + blocksdat_file.h bootstrap_serialization.h ) @@ -63,6 +67,15 @@ bitmonero_private_headers(blockchain_export ${blockchain_export_private_headers}) +set(blockchain_dump_sources + blockchain_dump.cpp + ) + +set(blockchain_dump_private_headers) + +bitmonero_private_headers(blockchain_dump + ${blockchain_dump_private_headers}) + if (BLOCKCHAIN_DB STREQUAL DB_LMDB) bitmonero_add_executable(blockchain_converter @@ -116,3 +129,21 @@ add_dependencies(blockchain_export set_property(TARGET blockchain_export PROPERTY OUTPUT_NAME "blockchain_export") + +bitmonero_add_executable(blockchain_dump + ${blockchain_dump_sources} + ${blockchain_dump_private_headers}) + +target_link_libraries(blockchain_dump + LINK_PRIVATE + cryptonote_core + blockchain_db + p2p + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_dump + version) +set_property(TARGET blockchain_dump + PROPERTY + OUTPUT_NAME "blockchain_dump") + diff --git a/src/blockchain_utilities/blockchain_dump.cpp b/src/blockchain_utilities/blockchain_dump.cpp new file mode 100644 index 000000000..4c3c09c43 --- /dev/null +++ b/src/blockchain_utilities/blockchain_dump.cpp @@ -0,0 +1,431 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/berkeleydb/db_bdb.h" +#include "blockchain_utilities.h" +#include "common/command_line.h" +#include "version.h" + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace po = boost::program_options; +using namespace epee; // log_space + +using namespace cryptonote; + +struct DumpContext { + std::ofstream &f; + size_t level; + std::vector<std::string> close; + + DumpContext(std::ofstream &f): f(f), level(0) {} +}; + +template<typename S> static void start_compound(DumpContext &d, S key, bool array, bool print = false) +{ + if (print) + LOG_PRINT_L0("Dumping " << key << "..."); + d.f << std::string(d.level*2, ' '); + d.f << "\"" << key << "\": " << (array ? "[" : "{") << " \n"; + d.close.push_back(array ? "]" : "}"); + d.level++; +} + +template<typename S> static void start_array(DumpContext &d, S key, bool print = false) { start_compound(d, key, true, print); } +template<typename S> static void start_struct(DumpContext &d, S key, bool print = false) { start_compound(d, key, false, print); } + +static void end_compound(DumpContext &d) +{ + d.level--; + d.f << std::string(d.level*2, ' '); + d.f << d.close.back() << ",\n"; + d.close.pop_back(); +} + +template<typename S, typename T> static void write_pod(DumpContext &d, S key, const T &t) +{ + d.f << std::string(d.level*2, ' '); + d.f << "\"" << key << "\": "; + d.f << t; + d.f << ",\n"; +} + +template<typename T> static void write_pod(DumpContext &d, const T &t) +{ + d.f << std::string(d.level*2, ' '); + d.f << t; + d.f << ",\n"; +} + +int main(int argc, char* argv[]) +{ + uint32_t log_level = 0; + uint64_t block_stop = 0; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true}; + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop}; +#if SOURCE_DB != DB_MEMORY + const command_line::arg_descriptor<std::string> arg_db_type = { + "db-type" + , "Specify database type" + , DEFAULT_DB_TYPE + }; + const command_line::arg_descriptor<bool> arg_include_db_only_data = { + "include-db-only-data" + , "Include data that is only in a database version." + , false + }; +#endif + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_output_file); +#if SOURCE_DB != DB_MEMORY + command_line::add_arg(desc_cmd_sett, arg_db_type); + command_line::add_arg(desc_cmd_sett, arg_include_db_only_data); +#endif + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_block_stop); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + log_level = command_line::get_arg(vm, arg_log_level); + block_stop = command_line::get_arg(vm, arg_block_stop); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + bool opt_testnet = command_line::get_arg(vm, arg_testnet_on); + bool opt_include_db_only_data = command_line::get_arg(vm, arg_include_db_only_data); + + std::string m_config_folder; + + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + m_config_folder = command_line::get_arg(vm, data_dir_arg); + + if (command_line::has_arg(vm, arg_output_file)) + output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); + else + output_file_path = boost::filesystem::path(m_config_folder) / "dump" / "blockchain.json"; + LOG_PRINT_L0("Export output file: " << output_file_path.string()); + + const boost::filesystem::path dir_path = output_file_path.parent_path(); + if (!dir_path.empty()) + { + if (boost::filesystem::exists(dir_path)) + { + if (!boost::filesystem::is_directory(dir_path)) + { + LOG_PRINT_RED_L0("dump directory path is a file: " << dir_path); + return false; + } + } + else + { + if (!boost::filesystem::create_directory(dir_path)) + { + LOG_PRINT_RED_L0("Failed to create directory " << dir_path); + return false; + } + } + } + + std::ofstream raw_data_file; + raw_data_file.open(output_file_path.string(), std::ios_base::out | std::ios::trunc); + if (raw_data_file.fail()) + return false; + + + // If we wanted to use the memory pool, we would set up a fake_core. + +#if SOURCE_DB == DB_MEMORY + // blockchain_storage* core_storage = NULL; + // tx_memory_pool m_mempool(*core_storage); // is this fake anyway? just passing in NULL! so m_mempool can't be used anyway, right? + // core_storage = new blockchain_storage(&m_mempool); + + blockchain_storage* core_storage = new blockchain_storage(NULL); + LOG_PRINT_L0("Initializing source blockchain (in-memory database)"); + r = core_storage->init(m_config_folder, opt_testnet); +#else + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + Blockchain *core_storage = NULL; + tx_memory_pool m_mempool(*core_storage); + core_storage = new Blockchain(m_mempool); + + BlockchainDB* db; + int mdb_flags = 0; + std::string db_type = command_line::get_arg(vm, arg_db_type); + if (db_type.empty() || db_type == "lmdb") + { + db = new BlockchainLMDB(); + mdb_flags |= MDB_RDONLY; + } + else if (db_type == "berkeley") + { + db = new BlockchainBDB(); + // can't open readonly due to the way flags are split in db_bdb.cpp + } + else + { + LOG_PRINT_L0("Invalid db type: " << db_type); + throw; + } + boost::filesystem::path folder(m_config_folder); + folder /= db->get_db_name(); + const std::string filename = folder.string(); + + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + try + { + db->open(filename, mdb_flags); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + throw; + } + r = core_storage->init(db, opt_testnet); +#endif + + CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + LOG_PRINT_L0("Dumping blockchain..."); + + DumpContext d(raw_data_file); + + start_struct(d,"blockchain"); + uint64_t height = core_storage->get_current_blockchain_height(); + write_pod(d, "height", height); +goto start; + start_array(d,"blockids", true); + for (uint64_t h = 0; h < height; ++h) + write_pod(d,core_storage->get_block_id_by_height(h)); + end_compound(d); + start_array(d,"txids", true); + { + std::vector<crypto::hash> txids; + core_storage->for_all_transactions([&txids](const crypto::hash &hash, const cryptonote::transaction &tx)->bool{txids.push_back(hash); return true;}); + std::sort(txids.begin(), txids.end(), + [](const crypto::hash &txid0, const crypto::hash &txid1) {return memcmp(txid0.data, txid1.data, sizeof(crypto::hash::data)) < 0;}); + for (size_t n = 0; n < txids.size(); ++n) + write_pod(d, txids[n]); + } + end_compound(d); + start_struct(d,"transactions", true); + { + for (uint64_t h = 0; h < height; ++h) + { + start_array(d, boost::lexical_cast<std::string>(h)); + std::list<cryptonote::block> blocks; + std::list<cryptonote::transaction> transactions, miner_tx; + core_storage->get_blocks(h, 1, blocks, transactions); + if (blocks.size() != 1) + throw std::string("Expected 1 block at height ") + boost::lexical_cast<std::string>(h); + crypto::hash txid = cryptonote::get_transaction_hash(blocks.front().miner_tx); + write_pod(d, string_tools::pod_to_hex(txid).c_str(), obj_to_json_str(blocks.front().miner_tx)); + std::vector<std::pair<crypto::hash, cryptonote::transaction>> txes; + for (std::list<cryptonote::transaction>::iterator i = transactions.begin(); i != transactions.end(); ++i) + txes.push_back(std::make_pair(cryptonote::get_transaction_hash(*i), *i)); + std::sort(txes.begin(), txes.end(), + [](const std::pair<crypto::hash, cryptonote::transaction> &tx0, const std::pair<crypto::hash, cryptonote::transaction> &tx1) {return memcmp(tx0.first.data, tx1.first.data, sizeof(crypto::key_image::data)) < 0;}); + for (std::vector<std::pair<crypto::hash, cryptonote::transaction>>::iterator i = txes.begin(); i != txes.end(); ++i) + { + write_pod(d, string_tools::pod_to_hex((*i).first).c_str(), obj_to_json_str((*i).second)); + } + end_compound(d); + } + } + start_struct(d,"blocks", true); + { + std::vector<crypto::hash> blockids; + core_storage->for_all_blocks([&](uint64_t height, const crypto::hash &hash, const cryptonote::block &b)->bool{ + start_struct(d, boost::lexical_cast<std::string>(height)); + write_pod(d, "hash", string_tools::pod_to_hex(hash)); + cryptonote::block block = b; + write_pod(d, "block", obj_to_json_str(block)); + end_compound(d); + return true; + }); + } + end_compound(d); + start_array(d,"key_images", true); + { + std::vector<crypto::key_image> key_images; + core_storage->for_all_key_images([&key_images](const crypto::key_image &k_image)->bool{key_images.push_back(k_image); return true;}); + std::sort(key_images.begin(), key_images.end(), + [](const crypto::key_image &k0, const crypto::key_image &k1) {return memcmp(k0.data, k1.data, sizeof(crypto::key_image::data)) < 0;}); + for (size_t n = 0; n < key_images.size(); ++n) + write_pod(d,key_images[n]); + } + end_compound(d); + if (opt_include_db_only_data) + { + start_struct(d, "block_timestamps", true); + for (uint64_t h = 0; h < height; ++h) + write_pod(d,boost::lexical_cast<std::string>(h),db->get_block_timestamp(h)); + end_compound(d); + start_struct(d, "block_difficulties", true); + for (uint64_t h = 0; h < height; ++h) + write_pod(d,boost::lexical_cast<std::string>(h),db->get_block_cumulative_difficulty(h)); + end_compound(d); + start_struct(d, "block_sizes", true); + for (uint64_t h = 0; h < height; ++h) + write_pod(d,boost::lexical_cast<std::string>(h),db->get_block_size(h)); + end_compound(d); + start_struct(d, "block_coins", true); + for (uint64_t h = 0; h < height; ++h) + write_pod(d,boost::lexical_cast<std::string>(h),db->get_block_already_generated_coins(h)); + end_compound(d); + start_struct(d, "block_heights", true); + for (uint64_t h = 0; h < height; ++h) + { + const crypto::hash hash = core_storage->get_block_id_by_height(h); + write_pod(d,boost::lexical_cast<std::string>(h),db->get_block_height(hash)); + } + end_compound(d); + { + std::vector<crypto::hash> txids; + core_storage->for_all_transactions([&txids](const crypto::hash &hash, const cryptonote::transaction &tx)->bool{txids.push_back(hash); return true;}); + std::sort(txids.begin(), txids.end(), + [](const crypto::hash &txid0, const crypto::hash &txid1) {return memcmp(txid0.data, txid1.data, sizeof(crypto::hash::data)) < 0;}); + start_struct(d, "transaction_unlock_times", true); + for (size_t n = 0; n < txids.size(); ++n) + write_pod(d,string_tools::pod_to_hex(txids[n]),db->get_tx_unlock_time(txids[n])); + end_compound(d); + start_struct(d, "transaction_block_heights", true); + for (size_t n = 0; n < txids.size(); ++n) + write_pod(d,string_tools::pod_to_hex(txids[n]),db->get_tx_block_height(txids[n])); + end_compound(d); + } + start_array(d, "tx_and_index_from_global", true); + for (uint64_t idx = 0; ; ++idx) + { + try + { + tx_out_index toi = db->get_output_tx_and_index_from_global(idx); + start_struct(d, boost::lexical_cast<std::string>(idx)); + write_pod(d, "tx_hash", string_tools::pod_to_hex(toi.first)); + write_pod(d, "tx_index", string_tools::pod_to_hex(toi.second)); + end_compound(d); + } + catch (const OUTPUT_DNE &) { break; } + } + end_compound(d); +start: + start_struct(d, "outputs_amounts", true); + for (uint64_t base = 1; base <= (uint64_t)10000000000000000000ul; base *= 10) for (uint64_t digit = 1; digit <= 9; ++digit) { + uint64_t amount = digit * base; + write_pod(d, boost::lexical_cast<std::string>(amount), db->get_num_outputs(amount)); + } + end_compound(d); + start_array(d, "output_keys", true); + for (uint64_t idx = 0; ; ++idx) + { + try + { + output_data_t od = db->get_output_key(idx); + start_struct(d, boost::lexical_cast<std::string>(idx)); + write_pod(d, "pubkey", string_tools::pod_to_hex(od.pubkey)); + write_pod(d, "unlock_time", od.unlock_time); + write_pod(d, "height", od.height); + end_compound(d); + } + catch (const OUTPUT_DNE &) { break; } + } + end_compound(d); + start_struct(d, "hf_versions", true); + for (uint64_t h = 0; h < height; ++h) + write_pod(d, boost::lexical_cast<std::string>(h), (unsigned int)db->get_hard_fork_version(h)); + end_compound(d); + start_struct(d, "hf_starting_heights", true); + for (unsigned int v = 0; v <= 255; ++v) + write_pod(d, boost::lexical_cast<std::string>(v), db->get_hard_fork_starting_height(v)); + end_compound(d); + } + end_compound(d); + + CHECK_AND_ASSERT_MES(r, false, "Failed to dump blockchain"); + + if (raw_data_file.fail()) + return false; + raw_data_file.flush(); + + LOG_PRINT_L0("Blockchain dump OK"); + + return 0; +} diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index 8db8b7cdb..4a96bad6b 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "bootstrap_file.h" +#include "blocksdat_file.h" #include "common/command_line.h" #include "version.h" @@ -39,6 +40,7 @@ int main(int argc, char* argv[]) { uint32_t log_level = 0; uint64_t block_stop = 0; + bool blocks_dat = false; boost::filesystem::path default_data_path {tools::get_default_data_dir()}; boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; @@ -54,6 +56,7 @@ int main(int argc, char* argv[]) , "Run on testnet." , false }; + const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat}; command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); @@ -62,6 +65,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_testnet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_block_stop); + command_line::add_arg(desc_cmd_sett, arg_blocks_dat); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -94,6 +98,7 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Setting log level = " << log_level); bool opt_testnet = command_line::get_arg(vm, arg_testnet_on); + bool opt_blocks_dat = command_line::get_arg(vm, arg_blocks_dat); std::string m_config_folder; @@ -155,8 +160,16 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Source blockchain storage initialized OK"); LOG_PRINT_L0("Exporting blockchain raw data..."); - BootstrapFile bootstrap; - r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop); + if (opt_blocks_dat) + { + BlocksdatFile blocksdat; + r = blocksdat.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop); + } + else + { + BootstrapFile bootstrap; + r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop); + } CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data"); LOG_PRINT_L0("Blockchain raw data exported OK"); } diff --git a/src/blockchain_utilities/blockchain_utilities.h b/src/blockchain_utilities/blockchain_utilities.h new file mode 100644 index 000000000..ff57a9320 --- /dev/null +++ b/src/blockchain_utilities/blockchain_utilities.h @@ -0,0 +1,51 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "version.h" + + +// CONFIG: choose one of the three #define's +// +// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows +// the exporter to use the in-memory blockchain while the other binaries +// work with LMDB, without recompiling anything. +// +#define SOURCE_DB DB_MEMORY +//#define SOURCE_DB DB_LMDB +// to use global compile-time setting (DB_MEMORY or DB_LMDB): +// #define SOURCE_DB BLOCKCHAIN_DB + + +// bounds checking is done before writing to buffer, but buffer size +// should be a sensible maximum +#define BUFFER_SIZE 1000000 +#define NUM_BLOCKS_PER_CHUNK 1 +#define BLOCKCHAIN_RAW "blockchain.raw" + diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp new file mode 100644 index 000000000..f7de2a480 --- /dev/null +++ b/src/blockchain_utilities/blocksdat_file.cpp @@ -0,0 +1,170 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blocksdat_file.h" + + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +namespace +{ + std::string refresh_string = "\r \r"; +} + + + +bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64_t block_stop) +{ + const boost::filesystem::path dir_path = file_path.parent_path(); + if (!dir_path.empty()) + { + if (boost::filesystem::exists(dir_path)) + { + if (!boost::filesystem::is_directory(dir_path)) + { + LOG_PRINT_RED_L0("export directory path is a file: " << dir_path); + return false; + } + } + else + { + if (!boost::filesystem::create_directory(dir_path)) + { + LOG_PRINT_RED_L0("Failed to create directory " << dir_path); + return false; + } + } + } + + m_raw_data_file = new std::ofstream(); + + uint64_t num_blocks = 0; + + LOG_PRINT_L0("creating file"); + + m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (m_raw_data_file->fail()) + return false; + + initialize_file(block_stop); + + return true; +} + + +bool BlocksdatFile::initialize_file(uint64_t block_stop) +{ + const uint32_t nblocks = block_stop + 1; + unsigned char nblocksc[4]; + + nblocksc[0] = nblocks & 0xff; + nblocksc[1] = (nblocks >> 8) & 0xff; + nblocksc[2] = (nblocks >> 16) & 0xff; + nblocksc[3] = (nblocks >> 24) & 0xff; + + // 4 bytes little endian + *m_raw_data_file << nblocksc[0]; + *m_raw_data_file << nblocksc[1]; + *m_raw_data_file << nblocksc[2]; + *m_raw_data_file << nblocksc[3]; + + return true; +} + +void BlocksdatFile::write_block(const crypto::hash& block_hash) +{ + const std::string data(block_hash.data, sizeof(block_hash)); + *m_raw_data_file << data; +} + +bool BlocksdatFile::close() +{ + if (m_raw_data_file->fail()) + return false; + + m_raw_data_file->flush(); + delete m_raw_data_file; + return true; +} + + +#if SOURCE_DB == DB_MEMORY +bool BlocksdatFile::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop) +#else +bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop) +#endif +{ + uint64_t num_blocks_written = 0; + m_blockchain_storage = _blockchain_storage; + uint64_t progress_interval = 100; + block b; + + uint64_t block_start = 0; + uint64_t block_stop = 0; + LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1); + if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height())) + { + LOG_PRINT_L0("Using requested block height: " << requested_block_stop); + block_stop = requested_block_stop; + } + else + { + block_stop = m_blockchain_storage->get_current_blockchain_height() - 1; + LOG_PRINT_L0("Using block height of source blockchain: " << block_stop); + } + LOG_PRINT_L0("Storing blocks raw data..."); + if (!BlocksdatFile::open_writer(output_file, block_stop)) + { + LOG_PRINT_RED_L0("failed to open raw file for write"); + return false; + } + for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height) + { + // this method's height refers to 0-based height (genesis block = height 0) + crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height); + write_block(hash); + if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) { + num_blocks_written += NUM_BLOCKS_PER_CHUNK; + } + if (m_cur_height % progress_interval == 0) { + std::cout << refresh_string; + std::cout << "block " << m_cur_height << "/" << block_stop << std::flush; + } + } + // print message for last block, which may not have been printed yet due to progress_interval + std::cout << refresh_string; + std::cout << "block " << m_cur_height-1 << "/" << block_stop << ENDL; + + LOG_PRINT_L0("Number of blocks exported: " << num_blocks_written); + + return BlocksdatFile::close(); +} + diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h new file mode 100644 index 000000000..eefef6fa6 --- /dev/null +++ b/src/blockchain_utilities/blocksdat_file.h @@ -0,0 +1,90 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/iostreams/stream_buffer.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/iostreams/device/back_inserter.hpp> + +#include <boost/iostreams/filtering_streambuf.hpp> + +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/cryptonote_boost_serialization.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +#include <algorithm> +#include <cstdio> +#include <fstream> +#include <boost/iostreams/copy.hpp> +#include <atomic> + +#include "common/command_line.h" +#include "version.h" + +#include "blockchain_utilities.h" + + +using namespace cryptonote; + + +class BlocksdatFile +{ +public: + +#if SOURCE_DB == DB_MEMORY + bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_file, uint64_t use_block_height=0); +#else + bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_file, uint64_t use_block_height=0); +#endif + +protected: + +#if SOURCE_DB == DB_MEMORY + blockchain_storage* m_blockchain_storage; +#else + Blockchain* m_blockchain_storage; +#endif + + std::ofstream * m_raw_data_file; + + // open export file for write + bool open_writer(const boost::filesystem::path& file_path, uint64_t block_stop); + bool initialize_file(uint64_t block_stop); + bool close(); + void write_block(const crypto::hash &block_hash); + +private: + + uint64_t m_cur_height; // tracks current height during export +}; diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index ab841c8c6..d66d2f604 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -53,20 +53,23 @@ namespace bool BootstrapFile::open_writer(const boost::filesystem::path& file_path) { const boost::filesystem::path dir_path = file_path.parent_path(); - if (boost::filesystem::exists(dir_path)) + if (!dir_path.empty()) { - if (!boost::filesystem::is_directory(dir_path)) + if (boost::filesystem::exists(dir_path)) { - LOG_PRINT_RED_L0("export directory path is a file: " << dir_path); - return false; + if (!boost::filesystem::is_directory(dir_path)) + { + LOG_PRINT_RED_L0("export directory path is a file: " << dir_path); + return false; + } } - } - else - { - if (!boost::filesystem::create_directory(dir_path)) + else { - LOG_PRINT_RED_L0("Failed to create directory " << dir_path); - return false; + if (!boost::filesystem::create_directory(dir_path)) + { + LOG_PRINT_RED_L0("Failed to create directory " << dir_path); + return false; + } } } diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index fcf89d1ac..6f8df18dd 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -49,24 +49,7 @@ #include "common/command_line.h" #include "version.h" - -// CONFIG: choose one of the three #define's -// -// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows -// the exporter to use the in-memory blockchain while the other binaries -// work with LMDB, without recompiling anything. -// -#define SOURCE_DB DB_MEMORY -// #define SOURCE_DB DB_LMDB -// to use global compile-time setting (DB_MEMORY or DB_LMDB): -// #define SOURCE_DB BLOCKCHAIN_DB - - -// bounds checking is done before writing to buffer, but buffer size -// should be a sensible maximum -#define BUFFER_SIZE 1000000 -#define NUM_BLOCKS_PER_CHUNK 1 -#define BLOCKCHAIN_RAW "blockchain.raw" +#include "blockchain_utilities.h" using namespace cryptonote; |