diff options
Diffstat (limited to 'src')
195 files changed, 17822 insertions, 841 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 43c5740af..e454b92fd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -89,11 +89,17 @@ endfunction () add_subdirectory(common) add_subdirectory(crypto) add_subdirectory(cryptonote_core) +add_subdirectory(blockchain_db) add_subdirectory(mnemonics) add_subdirectory(rpc) add_subdirectory(wallet) +add_subdirectory(p2p) +add_subdirectory(cryptonote_protocol) add_subdirectory(connectivity_tool) add_subdirectory(miner) add_subdirectory(simplewallet) +add_subdirectory(daemonizer) add_subdirectory(daemon) + +add_subdirectory(blockchain_converter) diff --git a/src/Changelog b/src/Changelog new file mode 100644 index 000000000..84da8068b --- /dev/null +++ b/src/Changelog @@ -0,0 +1,77 @@ + +[F] - fixes a bug from existing official version +[B] - important bug discovered in official version +[T] - testes +[n] - not used now (only in history), replaced or not accepted to master + +- adding support for user-local installed compiler (e.g. non-root users on older systems to use new compiler) +instructions to use monero on Debian 7 (build clang + deps) +- [n] faster build: cmake with fast build, and with cotire (precompiled headers), and limited targets +- fixed cmake local compiler e.g. BOOST_IGNORE_SYSTEM_PATHS after merges with the other new CMake + +- faster rebuild: separate out not-template base for main network classes: connection_basic.cpp class connection_basic +- faster rebuild: separate out hooks: connection_basic::do_send_handler_write() do_send_handler_write_from_queue() +- created library/cmake for the new no-templates-only networking code - libp2p + +- added command exit without DB save (only for testers) +- added option --no-igd to not wait for IGD on start (faster testing) + +- imported logging/debug tools from otshell, with macros like _note() _info() _warn() +- logging: added proper thread locking; showing thread number {1} in the output +- logging: also showing process number (PID) in short form (1,2,3,...) to debug forking daemon etc +- logging: fixed colors for normal windows text console (currently they do not work in msys windows text console) +- logging: added channels to have separate files created +- logging: option (compile-time) to debug the logging system itself +- logging: console colors work on windows/msys + +- created network throttle class + throttle manager +- cmdline options for network throttle class (limit-rate-up limit-rate-down limit-rate) +- option and command to in fact limit (outgoing) peers --out-peers out_peers [currently not working! after merge] [TODO] +- rpc commands for network throttle class (limit limit_up limit_down) +- setting ToS socket flag with option --tos-flag + +- connection type support, so that RPC connection is excluded from network limits + +- gather and save throughput statistics (speed vs limit) into data file +- statistics of details to tune implementation: sleep in network threads, number of peers +- optimized statistics: accumulate sum and limit disk writes + +(dr. monero code is not published in this commit, but prepared) +- dr. monero show the collected statistics (in real time) +- dr. monero many windows opened at once from predefined list +- dr. monero auto scale; selectable average window +- dr. monero colors, show both samples and average values (transparent) +- dr. monero showing current configured network limit (goal) +- dr. monero show date/git commit/comments for better info in screenshots +- dr. monero optimize speed and memory usage to handle files from long tests + +- [F] found few UBs in code like wrong initialization order +- [B] found and partial debugged deadlocks on existing program (faster testing) + +- [B] found locking problem with connection close race + +- debug option --test-drop-download to test no-DB version without running out of RAM +- also option --test-drop-download-heigh to start drop only after certain height + +- added again fast exit command (but it does not work too well yet. ONLY FOR DEVELOPERS!) + +- created new from-scratch clean network engine model (a new program/library) +- connected the logging, the network throttles and all code to network model +- [T] run experiments in fully controlled and very fast environment + +- fine tuned parameters of throttles (sleep, 3 window sizes) +- code to estimated current network-transfer size of block based get_avg_block_size() +- [T] tested sleep vs request-size limiting of downloads (request less or more then 200 blocks) +- for now we use sleep method + +- documented throttle classes and few other classes +- adjusted comment headers, copyrights, added the @monero tag to doxygen denoting monero maintainer/contact for not-monero code +- doxygen scripts fixed, tested and generated actual website + +- [n] rebased most of above when change to branch development was decided +- [n] again rebase when we returned to developing against branch master +- finall version was tested by developer on Windows 64-bit, Debian, Ubuntu +- fixed clock compilation problems for freebsd and mac os x; Fixed related problem again on mac os x. + + + diff --git a/src/blockchain_converter/CMakeLists.txt b/src/blockchain_converter/CMakeLists.txt new file mode 100644 index 000000000..660650980 --- /dev/null +++ b/src/blockchain_converter/CMakeLists.txt @@ -0,0 +1,115 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(blockchain_converter_sources + blockchain_converter.cpp + ) + +set(blockchain_converter_private_headers) + +bitmonero_private_headers(blockchain_converter + ${blockchain_converter_private_headers}) + +set(blockchain_import_sources + blockchain_import.cpp + ) + +set(blockchain_import_private_headers + import.h + fake_core.h + ) + +bitmonero_private_headers(blockchain_import + ${blockchain_import_private_headers}) + +set(blockchain_export_sources + blockchain_export.cpp + ) + +set(blockchain_export_private_headers + import.h + blockchain_export.h + ) + +bitmonero_private_headers(blockchain_export + ${blockchain_export_private_headers}) + + + +if (BLOCKCHAIN_DB STREQUAL DB_LMDB) +bitmonero_add_executable(blockchain_converter + ${blockchain_converter_sources} + ${blockchain_converter_private_headers}) + +target_link_libraries(blockchain_converter + LINK_PRIVATE + cryptonote_core + p2p + blockchain_db + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_converter + version) +set_property(TARGET blockchain_converter + PROPERTY + OUTPUT_NAME "blockchain_converter") +endif () + +bitmonero_add_executable(blockchain_import + ${blockchain_import_sources} + ${blockchain_import_private_headers}) + +target_link_libraries(blockchain_import + LINK_PRIVATE + cryptonote_core + blockchain_db + p2p + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_import + version) +set_property(TARGET blockchain_import + PROPERTY + OUTPUT_NAME "blockchain_import") + +bitmonero_add_executable(blockchain_export + ${blockchain_export_sources} + ${blockchain_export_private_headers}) + +target_link_libraries(blockchain_export + LINK_PRIVATE + cryptonote_core + p2p + blockchain_db + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_export + version) +set_property(TARGET blockchain_export + PROPERTY + OUTPUT_NAME "blockchain_export") diff --git a/src/blockchain_converter/README.md b/src/blockchain_converter/README.md new file mode 100644 index 000000000..00160c6b9 --- /dev/null +++ b/src/blockchain_converter/README.md @@ -0,0 +1,54 @@ + +For importing into the LMDB database, compile with `DATABASE=lmdb` + +e.g. + +`DATABASE=lmdb make release` + +This is also the default compile setting on the blockchain branch. + +By default, the exporter will use the original in-memory database (blockchain.bin) as its source. +This default is to make migrating to an LMDB database easy, without having to recompile anything. +To change the source, adjust `SOURCE_DB` in `src/blockchain_converter/blockchain_export.h` according to the comments. + +# Usage: + +See also each utility's "--help" option. + +## Export an existing in-memory database + +`$ blockchain_export` + +This loads the existing blockchain, for whichever database type it was compiled for, and exports it to `$MONERO_DATA_DIR/export/blockchain.raw` + +## Import the exported file + +`$ blockchain_import` + +This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database. + +Defaults: `--batch on`, `--batch size 20000`, `--verify on` + +Batch size refers to number of blocks and can be adjusted for performance based on available RAM. + +Verification should only be turned off if importing from a trusted blockchain. + +```bash +# use default settings to import blockchain.raw into database +$ blockchain_import + +# fast import with large batch size, verification off +$ blockchain_import --batch-size 100000 --verify off + +# LMDB flags can be set by appending them to the database type: +# flags: nosync, nometasync, writemap, mapasync +$ blockchain_import --database lmdb#nosync +$ blockchain_import --database lmdb#nosync,nometasync +``` + +## Blockchain converter with batching +`blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate. + +```bash +$ blockchain_converter --batch on --batch-size 20000 +``` diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp new file mode 100644 index 000000000..c3c6d0918 --- /dev/null +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -0,0 +1,299 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "common/util.h" +#include "warnings.h" +#include "crypto/crypto.h" +#include "cryptonote_config.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "misc_language.h" +#include "cryptonote_core/blockchain_storage.h" +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "cryptonote_core/tx_pool.h" +#include "common/command_line.h" +#include "serialization/json_utils.h" +#include "include_base_utils.h" +#include "version.h" +#include <iostream> + + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace +{ + +// CONFIG +bool opt_batch = true; +bool opt_resume = true; +bool opt_testnet = false; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +uint64_t db_batch_size = 20000; + +} + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +struct fake_core +{ + Blockchain dummy; + tx_memory_pool m_pool; + + blockchain_storage m_storage; + +#if !defined(BLOCKCHAIN_DB) + // for multi_db_runtime: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(&dummy), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(dummy), m_storage(&m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + } +}; + +int main(int argc, char* argv[]) +{ + uint64_t height = 0; + uint64_t start_block = 0; + uint64_t end_block = 0; + uint64_t num_blocks = 0; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", LOG_LEVEL_0}; + const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , opt_testnet + }; + const command_line::arg_descriptor<uint64_t> arg_block_number = + {"block-number", "Number of blocks (default: use entire source blockchain)", + 0}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_block_number); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + const command_line::arg_descriptor<bool> arg_batch = {"batch", + "Batch transactions for faster import", true}; + const command_line::arg_descriptor<bool> arg_resume = {"resume", + "Resume from current height if output database already exists", true}; + + // call add_options() directly for these arguments since command_line helpers + // support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + + return true; + }); + if (!r) + return 1; + + int log_level = command_line::get_arg(vm, arg_log_level); + opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + return 1; + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + return 1; + } + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + + std::string src_folder; + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + src_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path dest_folder(src_folder); + + num_blocks = command_line::get_arg(vm, arg_block_number); + + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + fake_core c(src_folder, opt_testnet); + + height = c.m_storage.get_current_blockchain_height(); + + BlockchainDB *blockchain; + blockchain = new BlockchainLMDB(opt_batch); + dest_folder /= blockchain->get_db_name(); + LOG_PRINT_L0("Source blockchain: " << src_folder); + LOG_PRINT_L0("Dest blockchain: " << dest_folder.string()); + LOG_PRINT_L0("Opening dest blockchain (BlockchainDB " << blockchain->get_db_name() << ")"); + blockchain->open(dest_folder.string()); + LOG_PRINT_L0("Source blockchain height: " << height); + LOG_PRINT_L0("Dest blockchain height: " << blockchain->height()); + + if (opt_resume) + // next block number to add is same as current height + start_block = blockchain->height(); + + if (! num_blocks || (start_block + num_blocks > height)) + end_block = height - 1; + else + end_block = start_block + num_blocks - 1; + + LOG_PRINT_L0("start height: " << start_block+1 << " stop height: " << + end_block+1); + + if (start_block > end_block) + { + LOG_PRINT_L0("Finished: no blocks to add"); + delete blockchain; + return 0; + } + + if (opt_batch) + blockchain->batch_start(); + uint64_t i = 0; + for (i = start_block; i < end_block + 1; ++i) + { + // block: i height: i+1 end height: end_block + 1 + if ((i+1) % 10 == 0) + { + std::cout << "\r \r" << "height " << i+1 << "/" << + end_block+1 << " (" << (i+1)*100/(end_block+1)<< "%)" << std::flush; + } + // for debugging: + // std::cout << "height " << i+1 << "/" << end_block+1 + // << " ((" << i+1 << ")*100/(end_block+1))" << "%)" << ENDL; + + block b = c.m_storage.get_block(i); + size_t bsize = c.m_storage.get_block_size(i); + difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); + uint64_t bcoins = c.m_storage.get_block_coins_generated(i); + std::vector<transaction> txs; + std::vector<crypto::hash> missed; + + c.m_storage.get_transactions(b.tx_hashes, txs, missed); + if (missed.size()) + { + std::cout << ENDL; + std::cerr << "Missed transaction(s) for block at height " << i + 1 << ", exiting" << ENDL; + delete blockchain; + return 1; + } + + try + { + blockchain->add_block(b, bsize, bdiff, bcoins, txs); + + if (opt_batch) + { + if ((i < end_block) && ((i + 1) % db_batch_size == 0)) + { + std::cout << "\r \r"; + std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL; + blockchain->batch_stop(); + blockchain->batch_start(); + std::cout << ENDL; + blockchain->show_stats(); + } + } + } + catch (const std::exception& e) + { + std::cout << ENDL; + std::cerr << "Error adding block " << i << " to new blockchain: " << e.what() << ENDL; + delete blockchain; + return 2; + } + } + if (opt_batch) + { + std::cout << "\r \r" << "height " << i << "/" << + end_block+1 << " (" << (i)*100/(end_block+1)<< "%)" << std::flush; + std::cout << ENDL; + std::cout << "[- batch commit at height " << i << " -]" << ENDL; + blockchain->batch_stop(); + } + std::cout << ENDL; + blockchain->show_stats(); + std::cout << "Finished at height: " << i << " block: " << i-1 << ENDL; + + delete blockchain; + return 0; +} diff --git a/src/blockchain_converter/blockchain_export.cpp b/src/blockchain_converter/blockchain_export.cpp new file mode 100644 index 000000000..dc5c7cadc --- /dev/null +++ b/src/blockchain_converter/blockchain_export.cpp @@ -0,0 +1,396 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <algorithm> +#include <cstdio> +#include <fstream> +#include <boost/iostreams/copy.hpp> +#include <atomic> + +#include <boost/archive/binary_oarchive.hpp> +#include <boost/archive/binary_iarchive.hpp> +#include <boost/iostreams/stream_buffer.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/iostreams/device/back_inserter.hpp> +#include <boost/iostreams/filtering_streambuf.hpp> +#include <boost/iostreams/filter/bzip2.hpp> +#include "common/command_line.h" +#include "version.h" +#include "blockchain_export.h" +#include "cryptonote_core/cryptonote_boost_serialization.h" + +#include "import.h" + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +static int max_chunk = 0; +static size_t height; + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +bool BlockchainExport::open(const boost::filesystem::path& dir_path) +{ + if (boost::filesystem::exists(dir_path)) + { + if (!boost::filesystem::is_directory(dir_path)) + { + LOG_PRINT_RED_L0("export directory path is a file: " << dir_path); + return false; + } + } + else + { + if (!boost::filesystem::create_directory(dir_path)) + { + LOG_PRINT_RED_L0("Failed to create directory " << dir_path); + return false; + } + } + + std::string file_path = (dir_path / BLOCKCHAIN_RAW).string(); + m_raw_data_file = new std::ofstream(); + m_raw_data_file->open(file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); + if (m_raw_data_file->fail()) + return false; + + m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer); + m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream); + if (m_raw_archive == NULL) + return false; + + return true; +} + +void BlockchainExport::flush_chunk() +{ + m_output_stream->flush(); + char buffer[STR_LENGTH_OF_INT + 1]; + int chunk_size = (int) m_buffer.size(); + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + } + sprintf(buffer, STR_FORMAT_OF_INT, chunk_size); + m_raw_data_file->write(buffer, STR_LENGTH_OF_INT); + if (max_chunk < chunk_size) + { + max_chunk = chunk_size; + } + long pos_before = m_raw_data_file->tellp(); + std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator<char>(*m_raw_data_file)); + m_raw_data_file->flush(); + long pos_after = m_raw_data_file->tellp(); + long num_chars_written = pos_after - pos_before; + if ((int) num_chars_written != chunk_size) + { + LOG_PRINT_RED_L0("INTERNAL ERROR: num chars wrote NEQ buffer size. height = " << height); + } + + m_buffer.clear(); + delete m_raw_archive; + delete m_output_stream; + m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer); + m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream); +} + +void BlockchainExport::serialize_block_to_text_buffer(const block& block) +{ + *m_raw_archive << block; +} + +void BlockchainExport::buffer_serialize_tx(const transaction& tx) +{ + *m_raw_archive << tx; +} + +void BlockchainExport::buffer_write_num_txs(const std::list<transaction> txs) +{ + int n = txs.size(); + *m_raw_archive << n; +} + +void BlockchainExport::write_block(block& block) +{ + serialize_block_to_text_buffer(block); + std::list<transaction> txs; + + uint64_t block_height = boost::get<txin_gen>(block.miner_tx.vin.front()).height; + + // put coinbase transaction first + transaction coinbase_tx = block.miner_tx; + crypto::hash coinbase_tx_hash = get_transaction_hash(coinbase_tx); +#if SOURCE_DB == DB_MEMORY + const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash); +#else + transaction cb_tx_full = m_blockchain_storage->get_db().get_tx(coinbase_tx_hash); +#endif + +#if SOURCE_DB == DB_MEMORY + if (cb_tx_full != NULL) + { + txs.push_back(*cb_tx_full); + } +#else + // TODO: should check and abort if cb_tx_full equals null_hash? + txs.push_back(cb_tx_full); +#endif + + // now add all regular transactions + BOOST_FOREACH(const auto& tx_id, block.tx_hashes) + { +#if SOURCE_DB == DB_MEMORY + const transaction* tx = m_blockchain_storage->get_tx(tx_id); +#else + transaction tx = m_blockchain_storage->get_db().get_tx(tx_id); +#endif + +#if SOURCE_DB == DB_MEMORY + if(tx == NULL) + { + if (! m_tx_pool) + throw std::runtime_error("Aborting: tx == NULL, so memory pool required to get tx, but memory pool isn't enabled"); + else + { + transaction tx; + if(m_tx_pool->get_transaction(tx_id, tx)) + txs.push_back(tx); + else + throw std::runtime_error("Aborting: tx not found in pool"); + } + } + else + txs.push_back(*tx); +#else + txs.push_back(tx); +#endif + } + + // serialize all txs to the persistant storage + buffer_write_num_txs(txs); + BOOST_FOREACH(const auto& tx, txs) + { + buffer_serialize_tx(tx); + } + + // These three attributes are currently necessary for a fast import that adds blocks without verification. + bool include_extra_block_data = true; + if (include_extra_block_data) + { +#if SOURCE_DB == DB_MEMORY + size_t block_size = m_blockchain_storage->get_block_size(block_height); +#else + size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); +#endif +#if SOURCE_DB == DB_MEMORY + difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height); +#else + difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); +#endif +#if SOURCE_DB == DB_MEMORY + uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height); +#else + // TODO TEST to verify that this is the equivalent. make sure no off-by-one error with block height vs block number + uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); +#endif + + *m_raw_archive << block_size; + *m_raw_archive << cumulative_difficulty; + *m_raw_archive << coins_generated; + } +} + +bool BlockchainExport::BlockchainExport::close() +{ + if (m_raw_data_file->fail()) + return false; + + m_raw_data_file->flush(); + delete m_raw_archive; + delete m_output_stream; + delete m_raw_data_file; + return true; +} + + +#if SOURCE_DB == DB_MEMORY +bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) +#else +bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) +#endif +{ + uint64_t block_height = 0; + m_blockchain_storage = _blockchain_storage; + m_tx_pool = _tx_pool; + uint64_t progress_interval = 100; + std::string refresh_string = "\r \r"; + LOG_PRINT_L0("Storing blocks raw data..."); + if (!BlockchainExport::open(output_dir)) + { + LOG_PRINT_RED_L0("failed to open raw file for write"); + return false; + } + block b; + LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()); + LOG_PRINT_L0("requested block height: " << requested_block_height); + if ((requested_block_height > 0) && (requested_block_height < m_blockchain_storage->get_current_blockchain_height())) + block_height = requested_block_height; + else + { + block_height = m_blockchain_storage->get_current_blockchain_height(); + LOG_PRINT_L0("Using block height of source blockchain: " << block_height); + } + for (height=0; height < block_height; ++height) + { + crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height); + m_blockchain_storage->get_block_by_hash(hash, b); + write_block(b); + if (height % NUM_BLOCKS_PER_CHUNK == 0) { + flush_chunk(); + } + if (height % progress_interval == 0) { + std::cout << refresh_string; + std::cout << "height " << height << "/" << block_height << std::flush; + } + } + if (height % NUM_BLOCKS_PER_CHUNK != 0) + { + flush_chunk(); + } + std::cout << refresh_string; + std::cout << "height " << height << "/" << block_height << ENDL; + + LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes"); + return BlockchainExport::close(); +} + + +int main(int argc, char* argv[]) +{ + uint32_t log_level = 0; + uint64_t block_height = 0; + std::string import_filename = BLOCKCHAIN_RAW; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor<uint64_t> arg_block_height = {"block-number", "", block_height}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_block_height); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + log_level = command_line::get_arg(vm, arg_log_level); + block_height = command_line::get_arg(vm, arg_block_height); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + bool opt_testnet = command_line::get_arg(vm, arg_testnet_on); + + std::string m_config_folder; + + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + m_config_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path output_dir {m_config_folder}; + output_dir /= "export"; + LOG_PRINT_L0("Export directory: " << output_dir.string()); + + // If we wanted to use the memory pool, we would set up a fake_core. + +#if SOURCE_DB == DB_MEMORY + // blockchain_storage* core_storage = NULL; + // tx_memory_pool m_mempool(*core_storage); // is this fake anyway? just passing in NULL! so m_mempool can't be used anyway, right? + // core_storage = new blockchain_storage(&m_mempool); + + blockchain_storage* core_storage = new blockchain_storage(NULL); + LOG_PRINT_L0("Initializing source blockchain (in-memory database)"); + r = core_storage->init(m_config_folder, opt_testnet); +#else + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + Blockchain* core_storage = NULL; + tx_memory_pool m_mempool(*core_storage); + core_storage = new Blockchain(m_mempool); + r = core_storage->init(m_config_folder, opt_testnet); +#endif + + CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + LOG_PRINT_L0("Exporting blockchain raw data..."); + + BlockchainExport be; + r = be.store_blockchain_raw(core_storage, NULL, output_dir, block_height); + CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data"); + LOG_PRINT_L0("Blockchain raw data exported OK"); +} diff --git a/src/blockchain_converter/blockchain_export.h b/src/blockchain_converter/blockchain_export.h new file mode 100644 index 000000000..43e25c039 --- /dev/null +++ b/src/blockchain_converter/blockchain_export.h @@ -0,0 +1,87 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/archive/binary_oarchive.hpp> +#include <boost/iostreams/stream_buffer.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/iostreams/device/back_inserter.hpp> +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +// CONFIG: choose one of the three #define's +// +// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows +// the exporter to use the in-memory blockchain while the other binaries +// work with LMDB, without recompiling anything. +// +#define SOURCE_DB DB_MEMORY +// #define SOURCE_DB DB_LMDB +// to use global compile-time setting (DB_MEMORY or DB_LMDB): +// #define SOURCE_DB BLOCKCHAIN_DB + +using namespace cryptonote; + +class BlockchainExport +{ +public: +#if SOURCE_DB == DB_MEMORY + bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#else + bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#endif + +protected: +#if SOURCE_DB == DB_MEMORY + blockchain_storage* m_blockchain_storage; +#else + Blockchain* m_blockchain_storage; +#endif + + tx_memory_pool* m_tx_pool; + typedef std::vector<char> buffer_type; + std::ofstream * m_raw_data_file; + boost::archive::binary_oarchive * m_raw_archive; + buffer_type m_buffer; + boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream; + + // open export file for write + bool open(const boost::filesystem::path& dir_path); + bool close(); + void write_block(block& block); + void serialize_block_to_text_buffer(const block& block); + void buffer_serialize_tx(const transaction& tx); + void buffer_write_num_txs(const std::list<transaction> txs); + void flush_chunk(); +}; diff --git a/src/blockchain_converter/blockchain_import.cpp b/src/blockchain_converter/blockchain_import.cpp new file mode 100644 index 000000000..6b432ccc6 --- /dev/null +++ b/src/blockchain_converter/blockchain_import.cpp @@ -0,0 +1,756 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <atomic> +#include <cstdio> +#include <algorithm> +#include <fstream> + +#include <boost/filesystem.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/archive/binary_iarchive.hpp> +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "cryptonote_core/cryptonote_boost_serialization.h" +#include "serialization/json_utils.h" // dump_json() +#include "include_base_utils.h" +#include "common/command_line.h" +#include "version.h" + +#include <lmdb.h> // for db flag arguments + +#include "import.h" +#include "fake_core.h" + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +// CONFIG +static bool opt_batch = true; +static bool opt_verify = true; // use add_new_block, which does verification before calling add_block +static bool opt_resume = true; +static bool opt_testnet = true; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +static uint64_t db_batch_size = 20000; + +static std::string refresh_string = "\r \r"; + + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +{ + std::vector<std::string> db_args; + boost::split(db_args, db_arg_str, boost::is_any_of("#")); + db_engine = db_args.front(); + boost::algorithm::trim(db_engine); + + if (db_args.size() == 1) + { + return 0; + } + else if (db_args.size() > 2) + { + std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL; + return 1; + } + + std::string db_arg_str2 = db_args[1]; + boost::split(db_args, db_arg_str2, boost::is_any_of(",")); + for (auto& it : db_args) + { + boost::algorithm::trim(it); + if (it.empty()) + continue; + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + { + mdb_flags |= MDB_NOSYNC; + } + else if (it == "nometasync") + { + mdb_flags |= MDB_NOMETASYNC; + } + else if (it == "writemap") + { + mdb_flags |= MDB_WRITEMAP; + } + else if (it == "mapasync") + { + mdb_flags |= MDB_MAPASYNC; + } + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } + return 0; +} + + +int count_blocks(std::string& import_file_path) +{ + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("import file not found: " << raw_file_path); + throw std::runtime_error("Aborting"); + } + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + throw std::runtime_error("Aborting"); + } + LOG_PRINT_L0("Scanning blockchain from import file..."); + char buffer1[STR_LENGTH_OF_INT + 1]; + block b; + transaction tx; + bool quit = false; + uint64_t bytes_read = 0; + int progress_interval = 10; + + while (! quit) + { + int chunk_size; + import_file.read(buffer1, STR_LENGTH_OF_INT); + if (!import_file) { + std::cout << refresh_string; + LOG_PRINT_L1("End of import file reached"); + quit = true; + break; + } + h += NUM_BLOCKS_PER_CHUNK; + if (h % progress_interval == 0) + { + std::cout << refresh_string << "block height: " << h << + std::flush; + } + bytes_read += STR_LENGTH_OF_INT; + buffer1[STR_LENGTH_OF_INT] = '\0'; + chunk_size = atoi(buffer1); + if (chunk_size > BUFFER_SIZE) + { + std::cout << refresh_string; + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE + << " height: " << h); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + std::cout << refresh_string; + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000" << " height: " + << h); + } + else if (chunk_size <= 0) { + std::cout << refresh_string; + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h); + throw std::runtime_error("Aborting"); + } + // skip to next expected block size value + import_file.seekg(chunk_size, std::ios_base::cur); + if (! import_file) { + std::cout << refresh_string; + LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + throw std::runtime_error("Aborting"); + } + bytes_read += chunk_size; + std::cout << refresh_string; + + LOG_PRINT_L3("Total bytes scanned: " << bytes_read); + } + + import_file.close(); + + std::cout << ENDL; + std::cout << "Done scanning import file" << ENDL; + std::cout << "Total bytes scanned: " << bytes_read << ENDL; + std::cout << "Height: " << h << ENDL; + + return h; +} + +template <typename FakeCore> +int import_from_file(FakeCore& simple_core, std::string& import_file_path) +{ +#if !defined(BLOCKCHAIN_DB) + static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value, + "FakeCore constraint error"); +#endif +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + if (std::is_same<fake_core_lmdb, FakeCore>::value) + { + // Reset stats, in case we're using newly created db, accumulating stats + // from addition of genesis block. + // This aligns internal db counts with importer counts. + simple_core.m_storage.get_db().reset_stats(); + } +#endif + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("import file not found: " << raw_file_path); + return false; + } + + uint64_t source_height = count_blocks(import_file_path); + LOG_PRINT_L0("import file blockchain height: " << source_height); + + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + return false; + } + char buffer1[STR_LENGTH_OF_INT + 1]; + char buffer_block[BUFFER_SIZE]; + block b; + transaction tx; + int quit = 0; + uint64_t bytes_read = 0; + + uint64_t start_height = 1; + if (opt_resume) + start_height = simple_core.m_storage.get_current_blockchain_height(); + + // Note that a new blockchain will start with a height of 1 (block number 0) + // due to genesis block being added at initialization. + + // CONFIG + // TODO: can expand on this, e.g. with --block-number option + uint64_t stop_height = source_height; + + // These are what we'll try to use, and they don't have to be a determination + // from source and destination blockchains, but those are the current + // defaults. + LOG_PRINT_L0("start height: " << start_height << " stop height: " << + stop_height); + + bool use_batch = false; + if (opt_batch) + { + if (simple_core.support_batch) + use_batch = true; + else + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + } + + if (use_batch) + simple_core.batch_start(); + + LOG_PRINT_L0("Reading blockchain from import file..."); + std::cout << ENDL; + + // Within the loop, we skip to start_height before we start adding. + // TODO: Not a bottleneck, but we can use what's done in count_blocks() and + // only do the chunk size reads, skipping the chunk content reads until we're + // at start_height. + while (! quit) + { + int chunk_size; + import_file.read(buffer1, STR_LENGTH_OF_INT); + if (! import_file) { + std::cout << refresh_string; + LOG_PRINT_L0("End of import file reached"); + quit = 1; + break; + } + bytes_read += STR_LENGTH_OF_INT; + buffer1[STR_LENGTH_OF_INT] = '\0'; + chunk_size = atoi(buffer1); + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000"); + } + else if (chunk_size < 0) { + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " < 0"); + return 2; + } + import_file.read(buffer_block, chunk_size); + if (! import_file) { + LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + return 2; + } + bytes_read += chunk_size; + LOG_PRINT_L3("Total bytes read: " << bytes_read); + + if (h + NUM_BLOCKS_PER_CHUNK < start_height + 1) + { + h += NUM_BLOCKS_PER_CHUNK; + continue; + } + if (h > stop_height) + { + LOG_PRINT_L0("Specified height reached - stopping. height: " << h << " block: " << h-1); + quit = 1; + break; + } + + try + { + boost::iostreams::basic_array_source<char> device(buffer_block, chunk_size); + boost::iostreams::stream<boost::iostreams::basic_array_source<char>> s(device); + boost::archive::binary_iarchive a(s); + + int display_interval = 1000; + int progress_interval = 10; + for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; chunk_ind++) + { + h++; + if (h % display_interval == 0) + { + std::cout << refresh_string; + LOG_PRINT_L0("loading block height " << h); + } + else + { + LOG_PRINT_L3("loading block height " << h); + } + try { + a >> b; + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("exception while de-archiving block, height=" << h); + quit = 1; + break; + } + LOG_PRINT_L2("block prev_id: " << b.prev_id << ENDL); + + if (h % progress_interval == 0) + { + std::cout << refresh_string << "block " << h-1 + << std::flush; + } + + std::vector<transaction> txs; + + int num_txs; + try + { + a >> num_txs; + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("exception while de-archiving tx-num, height=" << h); + quit = 1; + break; + } + for(int tx_num = 1; tx_num <= num_txs; tx_num++) + { + try { + a >> tx; + } + catch (const std::exception& e) + { + LOG_PRINT_RED_L0("exception while de-archiving tx, height=" << h <<", tx_num=" << tx_num); + quit = 1; + break; + } + // if (tx_num == 1) { + // std::cout << "coinbase transaction" << ENDL; + // } + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // NOTE: all tx hashes except for coinbase tx are available in the block data + // get_transaction_hash(tx, hsh, blob_size); + // LOG_PRINT_L0("tx " << tx_num << " " << hsh << " : " << ENDL); + // LOG_PRINT_L0(obj_to_json_str(tx) << ENDL); + + // add blocks with verification. + // for Blockchain and blockchain_storage add_new_block(). + if (opt_verify) + { + if (tx_num == 1) { + continue; // coinbase transaction. no need to insert to tx_pool. + } + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // get_transaction_hash(tx, hsh, blob_size); + tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool r = true; + r = simple_core.m_pool.add_tx(tx, tvc, true); + if (!r) + { + LOG_PRINT_RED_L0("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num); + quit = 1; + break; + } + } + else + { + // for add_block() method, without (much) processing. + // don't add coinbase transaction to txs. + // + // because add_block() calls + // add_transaction(blk_hash, blk.miner_tx) first, and + // then a for loop for the transactions in txs. + if (tx_num > 1) + { + txs.push_back(tx); + } + } + } + + if (opt_verify) + { + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + simple_core.m_storage.add_new_block(b, bvc); + + if (bvc.m_verifivation_failed) + { + LOG_PRINT_L0("Failed to add block to blockchain, verification failed, height = " << h); + LOG_PRINT_L0("skipping rest of import file"); + // ok to commit previously batched data because it failed only in + // verification of potential new block with nothing added to batch + // yet + quit = 1; + break; + } + if (! bvc.m_added_to_main_chain) + { + LOG_PRINT_L0("Failed to add block to blockchain, height = " << h); + LOG_PRINT_L0("skipping rest of import file"); + // make sure we don't commit partial block data + quit = 2; + break; + } + } + else + { + size_t block_size; + difficulty_type cumulative_difficulty; + uint64_t coins_generated; + + a >> block_size; + a >> cumulative_difficulty; + a >> coins_generated; + + std::cout << refresh_string; + LOG_PRINT_L2("block_size: " << block_size); + LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty); + LOG_PRINT_L2("coins_generated: " << coins_generated); + + try + { + simple_core.add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what()); + quit = 2; // make sure we don't commit partial block data + break; + } + } + + if (use_batch) + { + if (h % db_batch_size == 0) + { + std::cout << refresh_string; + std::cout << ENDL << "[- batch commit at height " << h << " -]" << ENDL; + simple_core.batch_stop(); + simple_core.batch_start(); + std::cout << ENDL; +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db().show_stats(); +#endif + } + } + } + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("exception while reading from import file, height=" << h); + return 2; + } + } // while + + import_file.close(); + + if (use_batch) + { + if (quit > 1) + { + // There was an error, so don't commit pending data. + // Destructor will abort write txn. + } + else + { + simple_core.batch_stop(); + } +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db().show_stats(); +#endif + if (h > 0) + LOG_PRINT_L0("Finished at height: " << h << " block: " << h-1); + } + std::cout << ENDL; + return 0; +} + +int main(int argc, char* argv[]) +{ + std::string import_filename = BLOCKCHAIN_RAW; +#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) + std::string default_db_engine = "memory"; +#else + std::string default_db_engine = "lmdb"; +#endif + + uint32_t log_level = LOG_LEVEL_0; + std::string dirname; + std::string db_arg_str; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + const command_line::arg_descriptor<bool> arg_count_blocks = { + "count-blocks" + , "Count blocks in import file and exit" + , false + }; + const command_line::arg_descriptor<std::string> arg_database = { + "database", "available: memory, lmdb" + , default_db_engine + }; + const command_line::arg_descriptor<bool> arg_verify = {"verify", + "Verify blocks and transactions during import", true}; + const command_line::arg_descriptor<bool> arg_batch = {"batch", + "Batch transactions for faster import", true}; + const command_line::arg_descriptor<bool> arg_resume = {"resume", + "Resume from current height if output database already exists", true}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_database); + + command_line::add_arg(desc_cmd_only, arg_count_blocks); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + // call add_options() directly for these arguments since + // command_line helpers support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + log_level = command_line::get_arg(vm, arg_log_level); + opt_verify = command_line::get_arg(vm, arg_verify); + opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + exit(1); + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + exit(1); + } + + std::vector<std::string> db_engines {"memory", "lmdb"}; + + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + dirname = command_line::get_arg(vm, data_dir_arg); + db_arg_str = command_line::get_arg(vm, arg_database); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + boost::filesystem::path file_path {dirname}; + std::string import_file_path; + + import_file_path = (file_path / "export" / import_filename).string(); + + if (command_line::has_arg(vm, arg_count_blocks)) + { + count_blocks(import_file_path); + exit(0); + } + + + std::string db_engine; + int mdb_flags = 0; + int res = 0; + res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + if (res) + { + std::cerr << "Error parsing database argument(s)" << ENDL; + exit(1); + } + + if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) + { + std::cerr << "Invalid database engine: " << db_engine << std::endl; + exit(1); + } + + LOG_PRINT_L0("database: " << db_engine); + LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + std::cout << "import file path: " << import_file_path << ENDL; + std::cout << "database path: " << file_path.string() << ENDL; + + try + { + + // fake_core needed for verification to work when enabled. + // + // NOTE: don't need fake_core method of doing things when we're going to call + // BlockchainDB add_block() directly and have available the 3 block + // properties to do so. Both ways work, but fake core isn't necessary in that + // circumstance. + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + if (db_engine == "lmdb") + { + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); + import_from_file(simple_core, import_file_path); + } + else if (db_engine == "memory") + { + fake_core_memory simple_core(dirname, opt_testnet); + import_from_file(simple_core, import_file_path); + } + else + { + std::cerr << "database engine unrecognized" << ENDL; + exit(1); + } + + // for multi_db_compile: +#else + if (db_engine != default_db_engine) + { + std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; + exit(1); + } +#if BLOCKCHAIN_DB == DB_LMDB + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); +#else + fake_core_memory simple_core(dirname, opt_testnet); +#endif + + import_from_file(simple_core, import_file_path); +#endif + + } + catch (const DB_ERROR& e) + { + std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL; + exit(1); + } + + // destructors called at exit: + // + // ensure db closed + // - transactions properly checked and handled + // - disk sync if needed + // + // fake_core object's destructor is called when it goes out of scope. For an + // LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn + // calls delete on its BlockchainDB derived class' object, which closes its + // files. +} diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h new file mode 100644 index 000000000..f82b05d04 --- /dev/null +++ b/src/blockchain_converter/fake_core.h @@ -0,0 +1,165 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/filesystem.hpp> +#include "cryptonote_core/blockchain.h" // BlockchainDB +#include "cryptonote_core/blockchain_storage.h" // in-memory DB +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +using namespace cryptonote; + + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB + +struct fake_core_lmdb +{ + Blockchain m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) + // for multi_db_compile: +#else + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) +#endif + { + m_pool.init(path.string()); + + BlockchainDB* db = new BlockchainLMDB(); + + boost::filesystem::path folder(path); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename, mdb_flags); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + throw; + } + + m_storage.init(db, use_testnet); + if (do_batch) + m_storage.get_db().set_batch_transactions(do_batch); + support_batch = true; + support_add_block = true; + } + ~fake_core_lmdb() + { + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) + { + return m_storage.get_db().add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + } + + void batch_start() + { + m_storage.get_db().batch_start(); + } + + void batch_stop() + { + m_storage.get_db().batch_stop(); + } + +}; +#endif + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_MEMORY + +struct fake_core_memory +{ + blockchain_storage m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(&m_storage), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(m_storage), m_storage(&m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + support_batch = false; + support_add_block = false; + } + ~fake_core_memory() + { + LOG_PRINT_L3("fake_core_memory() destructor called - want to see it ripple down"); + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) + { + // TODO: + // would need to refactor handle_block_to_main_chain() to have a direct add_block() method like Blockchain class + throw std::runtime_error("direct add_block() method not implemented for in-memory db"); + return 2; + } + + void batch_start() + { + LOG_PRINT_L0("WARNING: [batch_start] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + + void batch_stop() + { + LOG_PRINT_L0("WARNING: [batch_stop] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + +}; + +#endif diff --git a/src/blockchain_converter/import.h b/src/blockchain_converter/import.h new file mode 100644 index 000000000..632b4c0d9 --- /dev/null +++ b/src/blockchain_converter/import.h @@ -0,0 +1,37 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// TODO: bounds checking is done before writing to buffer, but buffer size +// should be a sensible maximum +#define BUFFER_SIZE 1000000 +#define NUM_BLOCKS_PER_CHUNK 1 +#define STR_LENGTH_OF_INT 9 +#define STR_FORMAT_OF_INT "%09d" +#define BLOCKCHAIN_RAW "blockchain.raw" diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt new file mode 100644 index 000000000..adbe804aa --- /dev/null +++ b/src/blockchain_db/CMakeLists.txt @@ -0,0 +1,76 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(blockchain_db_sources + blockchain_db.cpp + lmdb/db_lmdb.cpp + ) + +if (NOT STATIC) + set(blockchain_db_sources + ${blockchain_db_sources} + berkeleydb/db_bdb.cpp + ) +endif() + + +set(blockchain_db_headers) + +set(blockchain_db_private_headers + blockchain_db.h + lmdb/db_lmdb.h + ) + +if (NOT STATIC) + set(blockchain_db_private_headers + ${blockchain_db_private_headers} + berkeleydb/db_bdb.h + ) +endif() + +bitmonero_private_headers(blockchain_db + ${crypto_private_headers}) +bitmonero_add_library(blockchain_db + ${blockchain_db_sources} + ${blockchain_db_headers} + ${blockchain_db_private_headers}) +target_link_libraries(blockchain_db + LINK_PUBLIC + common + crypto + cryptonote_core + ${Boost_DATE_TIME_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SERIALIZATION_LIBRARY} + ${LMDB_LIBRARY} + ${BDB_LIBRARY} + LINK_PRIVATE + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp new file mode 100644 index 000000000..4b254500b --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -0,0 +1,1689 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_bdb.h" + +#include <boost/filesystem.hpp> +#include <memory> // std::unique_ptr +#include <cstring> // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace +{ + +template <typename T> +inline void throw0(const T &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +template <typename T> +inline void throw1(const T &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct bdb_cur +{ + bdb_cur(DbTxn* txn, Db* dbi) + { + if (dbi->cursor(txn, &m_cur, 0)) + throw0(cryptonote::DB_ERROR("Error opening db cursor")); + done = false; + } + + ~bdb_cur() { close(); } + + operator Dbc*() { return m_cur; } + operator Dbc**() { return &m_cur; } + Dbc* operator->() { return m_cur; } + + void close() + { + if (!done) + { + m_cur->close(); + done = true; + } + } + +private: + Dbc* m_cur; + bool done; +}; + +const char* const BDB_BLOCKS = "blocks"; +const char* const BDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const BDB_BLOCK_HEIGHTS = "block_heights"; +const char* const BDB_BLOCK_HASHES = "block_hashes"; +const char* const BDB_BLOCK_SIZES = "block_sizes"; +const char* const BDB_BLOCK_DIFFS = "block_diffs"; +const char* const BDB_BLOCK_COINS = "block_coins"; + +const char* const BDB_TXS = "txs"; +const char* const BDB_TX_UNLOCKS = "tx_unlocks"; +const char* const BDB_TX_HEIGHTS = "tx_heights"; +const char* const BDB_TX_OUTPUTS = "tx_outputs"; + +const char* const BDB_OUTPUT_TXS = "output_txs"; +const char* const BDB_OUTPUT_INDICES = "output_indices"; +const char* const BDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const BDB_OUTPUT_KEYS = "output_keys"; + +const char* const BDB_SPENT_KEYS = "spent_keys"; + +template<typename T> +struct Dbt_copy: public Dbt +{ + Dbt_copy(const T &t): t_copy(t) + { + init(); + } + + Dbt_copy() + { + init(); + } + + void init() + { + set_data(&t_copy); + set_size(sizeof(T)); + set_ulen(sizeof(T)); + set_flags(DB_DBT_USERMEM); + } + + operator T() + { + return t_copy; + } +private: + T t_copy; +}; + +template<> +struct Dbt_copy<cryptonote::blobdata>: public Dbt +{ + Dbt_copy(const cryptonote::blobdata &bd) : m_data(new char[bd.size()]) + { + memcpy(m_data.get(), bd.data(), bd.size()); + set_data(m_data.get()); + set_size(bd.size()); + set_ulen(bd.size()); + set_flags(DB_DBT_USERMEM); + } +private: + std::unique_ptr<char[]> m_data; +}; + +struct Dbt_safe : public Dbt +{ + Dbt_safe() + { + set_data(NULL); + set_flags(DB_DBT_MALLOC); + } + ~Dbt_safe() + { + void* buf = get_data(); + if (buf != NULL) + { + free(buf); + } + } +}; + +} // anonymous namespace + +namespace cryptonote +{ + +void BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::hash> val_h(blk_hash); + if (m_block_heights->exists(*m_write_txn, &val_h, 0) == 0) + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); + + if (m_height > 0) + { + Dbt_copy<crypto::hash> parent_key(blk.prev_id); + Dbt_copy<uint32_t> parent_h; + if (m_block_heights->get(*m_write_txn, &parent_key, &parent_h, 0)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + } + uint32_t parent_height = parent_h; + if (parent_height != m_height) + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); + } + + Dbt_copy<uint32_t> key(m_height + 1); + + Dbt_copy<blobdata> blob(block_to_blob(blk)); + auto res = m_blocks->put(*m_write_txn, &key, &blob, 0); + if (res) + throw0(DB_ERROR("Failed to add block blob to db transaction.")); + + Dbt_copy<size_t> sz(block_size); + if (m_block_sizes->put(*m_write_txn, &key, &sz, 0)) + throw0(DB_ERROR("Failed to add block size to db transaction.")); + + Dbt_copy<uint64_t> ts(blk.timestamp); + if (m_block_timestamps->put(*m_write_txn, &key, &ts, 0)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction.")); + + Dbt_copy<difficulty_type> diff(cumulative_difficulty); + if (m_block_diffs->put(*m_write_txn, &key, &diff, 0)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction.")); + + Dbt_copy<uint64_t> coinsgen(coins_generated); + if (m_block_coins->put(*m_write_txn, &key, &coinsgen, 0)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction.")); + + if (m_block_heights->put(*m_write_txn, &val_h, &key, 0)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction.")); + + if (m_block_hashes->put(*m_write_txn, &key, &val_h, 0)) + throw0(DB_ERROR("Failed to add block hash to db transaction.")); +} + +void BlockchainBDB::remove_block() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + + Dbt_copy<uint32_t> k(m_height); + Dbt_copy<crypto::hash> h; + if (m_block_hashes->get(*m_write_txn, &k, &h, 0)) + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + + if (m_blocks->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block to db transaction")); + + if (m_block_sizes->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + + if (m_block_diffs->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + + if (m_block_coins->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + + if (m_block_timestamps->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + + if (m_block_heights->del(*m_write_txn, &h, 0)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + + if (m_block_hashes->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); +} + +void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::hash> val_h(tx_hash); + + if (m_txs->exists(*m_write_txn, &val_h, 0) == 0) + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + + Dbt_copy<blobdata> blob(tx_to_blob(tx)); + if (m_txs->put(*m_write_txn, &val_h, &blob, 0)) + throw0(DB_ERROR("Failed to add tx blob to db transaction")); + + Dbt_copy<uint64_t> height(m_height + 1); + if (m_tx_heights->put(*m_write_txn, &val_h, &height, 0)) + throw0(DB_ERROR("Failed to add tx block height to db transaction")); + + Dbt_copy<uint64_t> unlock_time(tx.unlock_time); + if (m_tx_unlocks->put(*m_write_txn, &val_h, &unlock_time, 0)) + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); +} + +void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::hash> val_h(tx_hash); + if (m_txs->exists(*m_write_txn, &val_h, 0)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + + if (m_txs->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if (m_tx_unlocks->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); + if (m_tx_heights->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + + remove_tx_outputs(tx_hash, tx); + + if (m_tx_outputs->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); + +} + +void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<uint32_t> k(m_num_outputs + 1); + Dbt_copy<crypto::hash> v(tx_hash); + + if (m_output_txs->put(*m_write_txn, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); + if (m_tx_outputs->put(*m_write_txn, &v, &k, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy<uint64_t> val_local_index(local_index); + if (m_output_indices->put(*m_write_txn, &k, &val_local_index, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy<uint64_t> val_amount(tx_output.amount); + if (m_output_amounts->put(*m_write_txn, &val_amount, &k, 0)) + throw0(DB_ERROR("Failed to add output amount to db transaction.")); + + if (tx_output.target.type() == typeid(txout_to_key)) + { + Dbt_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); + if (m_output_keys->put(*m_write_txn, &k, &val_pubkey, 0)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + } + + m_num_outputs++; +} + +void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + bdb_cur cur(*m_write_txn, m_tx_outputs); + + Dbt_copy<crypto::hash> k(tx_hash); + Dbt_copy<uint32_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + throw0(DB_ERROR("Attempting to remove a tx's outputs, but none found.")); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + else + { + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + const tx_out tx_output = tx.vout[i]; + remove_output(v, tx_output.amount); + if (i < num_elems - 1) + { + cur->get(&k, &v, DB_NEXT_DUP); + } + } + } + + cur.close(); +} + +// TODO: probably remove this function +void BlockchainBDB::remove_output(const tx_out& tx_output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " (unused version - does nothing)"); + return; +} + +void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amount) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<uint32_t> k(out_index); + + auto result = m_output_indices->del(*m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } + + result = m_output_txs->del(*m_write_txn, &k, 0); + // if (result != 0 && result != DB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } + + result = m_output_keys->del(*m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + } + else if (result) + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + + remove_amount_output_index(amount, out_index); + + m_num_outputs--; +} + +void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_cur cur(*m_write_txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint32_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + goi = v; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = cur->del(0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str())); + } + else + { + // not found + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); +} + +void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::key_image> val_key(k_image); + if (m_spent_keys->exists(*m_write_txn, &val_key, 0) == 0) + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); + + Dbt_copy<char> val('\0'); + if (m_spent_keys->put(*m_write_txn, &val_key, &val, 0)) + throw1(DB_ERROR("Error adding spent key image to db transaction.")); +} + +void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::key_image> k(k_image); + auto result = m_spent_keys->del(*m_write_txn, &k, 0); + if (result != 0 && result != DB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of key image to db transaction")); +} + +blobdata BlockchainBDB::output_to_blob(const tx_out& output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + throw1(DB_ERROR("Error serializing output to blob")); + return b; +} + +tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + std::stringstream ss; + ss << blob; + binary_archive<false> ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + throw1(DB_ERROR("Error deserializing tx output blob")); + + return o; +} + +uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint32_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems; + cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return glob_index; +} + +void BlockchainBDB::check_open() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + +BlockchainBDB::~BlockchainBDB() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + if (m_open) + { + close(); + } +} + +BlockchainBDB::BlockchainBDB(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_height = 0; +} + +void BlockchainBDB::open(const std::string& filename, const int db_flags) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + if (m_open) + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + throw0(DB_OPEN_FAILURE("DB needs a directory path, but a file was passed")); + } + else + { + if (!boost::filesystem::create_directory(direc)) + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); + } + + m_folder = filename; + + try + { + + //Create BerkeleyDB environment + m_env = new DbEnv(0); // no flags needed for DbEnv + + uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK + | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER + | DB_THREAD; + + // last parameter left 0, files will be created with default rw access + m_env->open(filename.c_str(), db_env_open_flags, 0); + + // begin transaction to init dbs + bdb_txn_safe txn; + m_env->txn_begin(NULL, txn, 0); + + // create Dbs in the environment + m_blocks = new Db(m_env, 0); + m_block_heights = new Db(m_env, 0); + m_block_hashes = new Db(m_env, 0); + m_block_timestamps = new Db(m_env, 0); + m_block_sizes = new Db(m_env, 0); + m_block_diffs = new Db(m_env, 0); + m_block_coins = new Db(m_env, 0); + + m_txs = new Db(m_env, 0); + m_tx_unlocks = new Db(m_env, 0); + m_tx_heights = new Db(m_env, 0); + m_tx_outputs = new Db(m_env, 0); + + m_output_txs = new Db(m_env, 0); + m_output_indices = new Db(m_env, 0); + m_output_amounts = new Db(m_env, 0); + m_output_keys = new Db(m_env, 0); + + m_spent_keys = new Db(m_env, 0); + + // Tell DB about Dbs that need duplicate support + // Note: no need to tell about sorting, + // as the default is insertion order, which we want + m_tx_outputs->set_flags(DB_DUP); + m_output_amounts->set_flags(DB_DUP); + + // Tell DB about fixed-size values. + m_block_hashes->set_re_len(sizeof(crypto::hash)); + m_block_timestamps->set_re_len(sizeof(uint64_t)); + m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t... + m_block_diffs->set_re_len(sizeof(difficulty_type)); + m_block_coins->set_re_len(sizeof(uint64_t)); + + m_output_txs->set_re_len(sizeof(crypto::hash)); + m_output_indices->set_re_len(sizeof(uint64_t)); + m_output_keys->set_re_len(sizeof(crypto::public_key)); + + //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) + // for the RECNO databases. We shouldn't as we're only + // inserting/removing from the end, but we'll see. + + // open Dbs in the environment + // m_tx_outputs and m_output_amounts must be DB_HASH or DB_BTREE + // because they need duplicate entry support. The rest are DB_RECNO, + // as it seems that will be the most performant choice. + m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0); + + m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); + m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0); + + m_txs->open(txn, BDB_TXS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0); + + m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0); + m_output_indices->open(txn, BDB_OUTPUT_INDICES, NULL, DB_RECNO, DB_CREATE, 0); + m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0); + m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + + m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_HASH, DB_CREATE, 0); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_blocks->stat(txn, &stats, DB_FAST_STAT); + m_height = stats->bt_nkeys; + delete stats; + + // see above comment about DB_FAST_STAT + m_output_indices->stat(txn, &stats, DB_FAST_STAT); + m_num_outputs = stats->bt_nkeys; + delete stats; + + txn.commit(); + } + catch (const std::exception& e) + { + throw0(DB_OPEN_FAILURE(e.what())); + } + + m_open = true; +} + +void BlockchainBDB::close() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + this->sync(); + + // FIXME: not yet thread safe!!! Use with care. + m_open = false; + m_env->close(DB_FORCESYNC); +} + +void BlockchainBDB::sync() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + try + { + m_blocks->sync(0); + m_block_heights->sync(0); + m_block_hashes->sync(0); + m_block_timestamps->sync(0); + m_block_sizes->sync(0); + m_block_diffs->sync(0); + m_block_coins->sync(0); + + m_txs->sync(0); + m_tx_unlocks->sync(0); + m_tx_heights->sync(0); + m_tx_outputs->sync(0); + + m_output_txs->sync(0); + m_output_indices->sync(0); + m_output_amounts->sync(0); + m_output_keys->sync(0); + + m_spent_keys->sync(0); + } + catch (const std::exception& e) + { + throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()).c_str())); + } +} + +void BlockchainBDB::reset() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // TODO: this +} + +std::vector<std::string> BlockchainBDB::get_filenames() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + std::vector<std::string> filenames; + + char *fname, *dbname; + const char **pfname, **pdbname; + + pfname = (const char **)&fname; + pdbname = (const char **)&dbname; + + m_blocks->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_heights->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_hashes->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_timestamps->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_sizes->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_diffs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_coins->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_txs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_tx_unlocks->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_tx_heights->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_tx_outputs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_txs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_indices->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_amounts->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_keys->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_spent_keys->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + std::vector<std::string> full_paths; + + for (auto& filename : filenames) + { + boost::filesystem::path p(m_folder); + p /= filename; + full_paths.push_back(p.string()); + } + + return full_paths; +} + +std::string BlockchainBDB::get_db_name() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + return std::string("BerkeleyDB"); +} + +// TODO: this? +bool BlockchainBDB::lock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + return false; +} + +// TODO: this? +void BlockchainBDB::unlock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +bool BlockchainBDB::block_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + + auto get_result = m_block_heights->exists(txn, &key, 0); + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + + txn.commit(); + return true; +} + +block BlockchainBDB::get_block(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + return get_block_from_height(get_block_height(h)); +} + +uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_copy<uint64_t> result; + + auto get_result = m_block_heights->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + + txn.commit(); + + return (uint64_t)result - 1; +} + +block_header BlockchainBDB::get_block_header(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + // block_header object is automatically cast from block object + return get_block(h); +} + +block BlockchainBDB::get_block_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_safe result; + auto get_result = m_blocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size()); + + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + return b; +} + +uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<uint64_t> result; + auto get_result = m_block_timestamps->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); + + txn.commit(); + return result; +} + +uint64_t BlockchainBDB::get_top_block_timestamp() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); +} + +size_t BlockchainBDB::get_block_size(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<size_t> result; + auto get_result = m_block_sizes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); + + txn.commit(); + return result; +} + +difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<difficulty_type> result; + auto get_result = m_block_diffs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + txn.commit(); + return result; +} + +difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; +} + +uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<uint64_t> result; + auto get_result = m_block_coins->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); + + txn.commit(); + return result; +} + +crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<crypto::hash> result; + auto get_result = m_block_hashes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block hash from the db.")); + + txn.commit(); + return result; +} + +std::vector<block> BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<block> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + + return v; +} + +std::vector<crypto::hash> BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<crypto::hash> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + + return v; +} + +crypto::hash BlockchainBDB::top_block_hash() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + if (m_height > 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; +} + +block BlockchainBDB::get_top_block() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + if (m_height > 0) + { + return get_block_from_height(m_height - 1); + } + + block b; + return b; +} + +uint64_t BlockchainBDB::height() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + return m_height; +} + +bool BlockchainBDB::tx_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + + TIME_MEASURE_START(time1); + auto get_result = m_txs->exists(txn, &key, 0); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + + return true; +} + +uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_copy<uint64_t> result; + auto get_result = m_tx_unlocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + + return result; +} + +transaction BlockchainBDB::get_tx(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_safe result; + auto get_result = m_txs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size()); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + txn.commit(); + + return tx; +} + +uint64_t BlockchainBDB::get_tx_count() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_txs->stat(txn, &stats, DB_FAST_STAT); + auto num_txs = stats->bt_nkeys; + delete stats; + + txn.commit(); + + return num_txs; +} + +std::vector<transaction> BlockchainBDB::get_tx_list(const std::vector<crypto::hash>& hlist) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<transaction> v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + + return v; +} + +uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_copy<uint64_t> result; + auto get_result = m_tx_heights->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + + txn.commit(); + + return (uint64_t)result - 1; +} + +//FIXME: make sure the random method used here is appropriate +uint64_t BlockchainBDB::get_random_output(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); + + return crypto::rand<uint64_t>() % num_outputs; +} + +uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint64_t> v; + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + return 0; + } + else if (result) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + txn.commit(); + + return num_elems; +} + +crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> k(glob_index); + Dbt_copy<crypto::public_key> v; + auto get_result = m_output_keys->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + return v; +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + return tx_out(); +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainBDB::get_output(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + return tx_out(); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> k(index); + Dbt_copy<crypto::hash > v; + + auto get_result = m_output_txs->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = v; + + Dbt_copy<uint64_t> result; + get_result = m_output_indices->get(txn, &k, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + + txn.commit(); + + return tx_out_index(tx_hash, result); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint64_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return get_output_tx_and_index_from_global(glob_index); +} + +std::vector<uint64_t> BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_tx_outputs); + + Dbt_copy<crypto::hash> k(h); + Dbt_copy<uint64_t> v; + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + index_vec.push_back(v); + cur->get(&k, &v, DB_NEXT_DUP); + } + + cur.close(); + txn.commit(); + + return index_vec; +} + +std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + std::vector<uint64_t> index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + for (const auto& vout : tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint64_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + output_index = v; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; +} + + + +bool BlockchainBDB::has_key_image(const crypto::key_image& img) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::key_image> val_key(img); + if (m_spent_keys->exists(txn, &val_key, 0) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); + return false; +} + +// Ostensibly BerkeleyDB has batch transaction support built-in, +// so the following few functions will be NOP. + +void BlockchainBDB::batch_start() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + +uint64_t BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + m_write_txn = NULL; + + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } + catch (const std::exception& e) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + return ++m_height; +} + +void BlockchainBDB::pop_block(block& blk, std::vector<transaction>& txs) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + + m_write_txn = NULL; + txn.commit(); + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + --m_height; +} + +} // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h new file mode 100644 index 000000000..d4eb5434c --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -0,0 +1,288 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <db_cxx.h> + +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata + +namespace cryptonote +{ + +struct bdb_txn_safe +{ + bdb_txn_safe() : m_txn(NULL) { } + ~bdb_txn_safe() + { + LOG_PRINT_L3("bdb_txn_safe: destructor"); + + if (m_txn != NULL) + abort(); + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (m_txn->commit(0)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + void abort() + { + LOG_PRINT_L3("bdb_txn_safe: abort()"); + if(m_txn != NULL) + { + m_txn->abort(); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: bdb_txn_safe: abort() called, but m_txn is NULL"); + } + } + + operator DbTxn*() + { + return m_txn; + } + + operator DbTxn**() + { + return &m_txn; + } + + DbTxn* m_txn; +}; + +class BlockchainBDB : public BlockchainDB +{ +public: + BlockchainBDB(bool batch_transactions=false); + ~BlockchainBDB(); + + virtual void open(const std::string& filename, const int db_flags); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector<std::string> get_filenames() const; + + virtual std::string get_db_name() const; + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h) const; + + virtual block get_block(const crypto::hash& h) const; + + virtual uint64_t get_block_height(const crypto::hash& h) const; + + virtual block_header get_block_header(const crypto::hash& h) const; + + virtual block get_block_from_height(const uint64_t& height) const; + + virtual uint64_t get_block_timestamp(const uint64_t& height) const; + + virtual uint64_t get_top_block_timestamp() const; + + virtual size_t get_block_size(const uint64_t& height) const; + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; + + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; + + virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual crypto::hash top_block_hash() const; + + virtual block get_top_block() const; + + virtual uint64_t height() const; + + virtual bool tx_exists(const crypto::hash& h) const; + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; + + virtual transaction get_tx(const crypto::hash& h) const; + + virtual uint64_t get_tx_count() const; + + virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const; + + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; + + virtual uint64_t get_random_output(const uint64_t& amount) const; + + virtual uint64_t get_num_outputs(const uint64_t& amount) const; + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; + + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; + + virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; + + virtual bool has_key_image(const crypto::key_image& img) const; + + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ); + + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + + virtual void pop_block(block& blk, std::vector<transaction>& txs); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& block_hash + ); + + virtual void remove_block(); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob) const; + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + + void check_open() const; + + DbEnv* m_env; + + Db* m_blocks; + Db* m_block_heights; + Db* m_block_hashes; + Db* m_block_timestamps; + Db* m_block_sizes; + Db* m_block_diffs; + Db* m_block_coins; + + Db* m_txs; + Db* m_tx_unlocks; + Db* m_tx_heights; + Db* m_tx_outputs; + + Db* m_output_txs; + Db* m_output_indices; + Db* m_output_amounts; + Db* m_output_keys; + + Db* m_spent_keys; + + uint64_t m_height; + uint64_t m_num_outputs; + std::string m_folder; + bdb_txn_safe *m_write_txn; + + bool m_batch_transactions; // support for batch transactions +}; + +} // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp new file mode 100644 index 000000000..d648be44e --- /dev/null +++ b/src/blockchain_db/blockchain_db.cpp @@ -0,0 +1,185 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blockchain_db.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace cryptonote +{ + +void BlockchainDB::pop_block() +{ + block blk; + std::vector<transaction> txs; + pop_block(blk, txs); +} + +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) +{ + crypto::hash tx_hash; + if (!tx_hash_ptr) + { + // should only need to compute hash for miner transactions + tx_hash = get_transaction_hash(tx); + LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash); + } + else + { + tx_hash = *tx_hash_ptr; + } + + add_transaction_data(blk_hash, tx, tx_hash); + + // iterate tx.vout using indices instead of C++11 foreach syntax because + // we need the index + if (tx.vout.size() != 0) // it may be technically possible for a tx to have no outputs + { + for (uint64_t i = 0; i < tx.vout.size(); ++i) + { + add_output(tx_hash, tx.vout[i], i); + } + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + add_spent_key(boost::get<txin_to_key>(tx_input).k_image); + } + } + } +} + +uint64_t BlockchainDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) +{ + TIME_MEASURE_START(time1); + crypto::hash blk_hash = get_block_hash(blk); + TIME_MEASURE_FINISH(time1); + time_blk_hash += time1; + + // call out to subclass implementation to add the block & metadata + time1 = epee::misc_utils::get_tick_count(); + add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); + TIME_MEASURE_FINISH(time1); + time_add_block1 += time1; + + // call out to add the transactions + + time1 = epee::misc_utils::get_tick_count(); + add_transaction(blk_hash, blk.miner_tx); + int tx_i = 0; + crypto::hash tx_hash = null_hash; + for (const transaction& tx : txs) + { + tx_hash = blk.tx_hashes[tx_i]; + add_transaction(blk_hash, tx, &tx_hash); + ++tx_i; + } + TIME_MEASURE_FINISH(time1); + time_add_transaction += time1; + + ++num_calls; + + return height(); +} + +void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs) +{ + blk = get_top_block(); + + remove_block(); + + remove_transaction(get_transaction_hash(blk.miner_tx)); + for (const auto& h : blk.tx_hashes) + { + txs.push_back(get_tx(h)); + remove_transaction(h); + } +} + +bool BlockchainDB::is_open() +{ + return m_open; +} + +void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) +{ + transaction tx = get_tx(tx_hash); + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + remove_spent_key(boost::get<txin_to_key>(tx_input).k_image); + } + } + + // need tx as tx.vout has the tx outputs, and the output amounts are needed + remove_transaction_data(tx_hash, tx); +} + +void BlockchainDB::reset_stats() +{ + num_calls = 0; + time_blk_hash = 0; + time_tx_exists = 0; + time_add_block1 = 0; + time_add_transaction = 0; + time_commit1 = 0; +} + +void BlockchainDB::show_stats() +{ + LOG_PRINT_L1(ENDL + << "*********************************" + << ENDL + << "num_calls: " << num_calls + << ENDL + << "time_blk_hash: " << time_blk_hash << "ms" + << ENDL + << "time_tx_exists: " << time_tx_exists << "ms" + << ENDL + << "time_add_block1: " << time_add_block1 << "ms" + << ENDL + << "time_add_transaction: " << time_add_transaction << "ms" + << ENDL + << "time_commit1: " << time_commit1 << "ms" + << ENDL + << "*********************************" + << ENDL + ); +} + +} // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h new file mode 100644 index 000000000..04d9c5384 --- /dev/null +++ b/src/blockchain_db/blockchain_db.h @@ -0,0 +1,492 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#ifndef BLOCKCHAIN_DB_H +#define BLOCKCHAIN_DB_H + +#include <list> +#include <string> +#include <exception> +#include "crypto/hash.h" +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/difficulty.h" + +/* DB Driver Interface + * + * The DB interface is a store for the canonical block chain. + * It serves as a persistent storage for the blockchain. + * + * For the sake of efficiency, the reference implementation will also + * store some blockchain data outside of the blocks, such as spent + * transfer key images, unspent transaction outputs, etc. + * + * Transactions are duplicated so that we don't have to fetch a whole block + * in order to fetch a transaction from that block. If this is deemed + * unnecessary later, this can change. + * + * Spent key images are duplicated outside of the blocks so it is quick + * to verify an output hasn't already been spent + * + * Unspent transaction outputs are duplicated to quickly gather random + * outputs to use for mixins + * + * IMPORTANT: + * A concrete implementation of this interface should populate these + * duplicated members! It is possible to have a partial implementation + * of this interface call to private members of the interface to be added + * later that will then populate as needed. + * + * General: + * open() + * is_open() + * close() + * sync() + * reset() + * + * Lock and unlock provided for reorg externally, and for block + * additions internally, this way threaded reads are completely fine + * unless the blockchain is changing. + * bool lock() + * unlock() + * + * vector<str> get_filenames() + * + * Blocks: + * bool block_exists(hash) + * height add_block(block, block_size, cumulative_difficulty, coins_generated, transactions) + * block get_block(hash) + * height get_block_height(hash) + * header get_block_header(hash) + * block get_block_from_height(height) + * size_t get_block_size(height) + * difficulty get_block_cumulative_difficulty(height) + * uint64_t get_block_already_generated_coins(height) + * uint64_t get_block_timestamp(height) + * uint64_t get_top_block_timestamp() + * hash get_block_hash_from_height(height) + * blocks get_blocks_range(height1, height2) + * hashes get_hashes_range(height1, height2) + * hash top_block_hash() + * block get_top_block() + * height height() + * void pop_block(block&, tx_list&) + * + * Transactions: + * bool tx_exists(hash) + * uint64_t get_tx_unlock_time(hash) + * tx get_tx(hash) + * uint64_t get_tx_count() + * tx_list get_tx_list(hash_list) + * height get_tx_block_height(hash) + * + * Outputs: + * index get_random_output(amount) + * uint64_t get_num_outputs(amount) + * pub_key get_output_key(amount, index) + * tx_out get_output(tx_hash, index) + * hash,index get_output_tx_and_index_from_global(index) + * hash,index get_output_tx_and_index(amount, index) + * vec<uint64> get_tx_output_indices(tx_hash) + * + * + * Spent Output Key Images: + * bool has_key_image(key_image) + * + * Exceptions: + * DB_ERROR -- generic + * DB_OPEN_FAILURE + * DB_CREATE_FAILURE + * DB_SYNC_FAILURE + * BLOCK_DNE + * BLOCK_PARENT_DNE + * BLOCK_EXISTS + * BLOCK_INVALID -- considering making this multiple errors + * TX_DNE + * TX_EXISTS + * OUTPUT_DNE + * OUTPUT_EXISTS + * KEY_IMAGE_EXISTS + */ + +namespace cryptonote +{ + +// typedef for convenience +typedef std::pair<crypto::hash, uint64_t> tx_out_index; + +/*********************************** + * Exception Definitions + ***********************************/ +class DB_EXCEPTION : public std::exception +{ + private: + std::string m; + + protected: + DB_EXCEPTION(const char *s) : m(s) { } + + public: + virtual ~DB_EXCEPTION() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class DB_ERROR : public DB_EXCEPTION +{ + public: + DB_ERROR() : DB_EXCEPTION("Generic DB Error") { } + DB_ERROR(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_OPEN_FAILURE : public DB_EXCEPTION +{ + public: + DB_OPEN_FAILURE() : DB_EXCEPTION("Failed to open the db") { } + DB_OPEN_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_CREATE_FAILURE : public DB_EXCEPTION +{ + public: + DB_CREATE_FAILURE() : DB_EXCEPTION("Failed to create the db") { } + DB_CREATE_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_SYNC_FAILURE : public DB_EXCEPTION +{ + public: + DB_SYNC_FAILURE() : DB_EXCEPTION("Failed to sync the db") { } + DB_SYNC_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_DNE : public DB_EXCEPTION +{ + public: + BLOCK_DNE() : DB_EXCEPTION("The block requested does not exist") { } + BLOCK_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_PARENT_DNE : public DB_EXCEPTION +{ + public: + BLOCK_PARENT_DNE() : DB_EXCEPTION("The parent of the block does not exist") { } + BLOCK_PARENT_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_EXISTS : public DB_EXCEPTION +{ + public: + BLOCK_EXISTS() : DB_EXCEPTION("The block to be added already exists!") { } + BLOCK_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_INVALID : public DB_EXCEPTION +{ + public: + BLOCK_INVALID() : DB_EXCEPTION("The block to be added did not pass validation!") { } + BLOCK_INVALID(const char* s) : DB_EXCEPTION(s) { } +}; + +class TX_DNE : public DB_EXCEPTION +{ + public: + TX_DNE() : DB_EXCEPTION("The transaction requested does not exist") { } + TX_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class TX_EXISTS : public DB_EXCEPTION +{ + public: + TX_EXISTS() : DB_EXCEPTION("The transaction to be added already exists!") { } + TX_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class OUTPUT_DNE : public DB_EXCEPTION +{ + public: + OUTPUT_DNE() : DB_EXCEPTION("The output requested does not exist!") { } + OUTPUT_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class OUTPUT_EXISTS : public DB_EXCEPTION +{ + public: + OUTPUT_EXISTS() : DB_EXCEPTION("The output to be added already exists!") { } + OUTPUT_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class KEY_IMAGE_EXISTS : public DB_EXCEPTION +{ + public: + KEY_IMAGE_EXISTS() : DB_EXCEPTION("The spent key image to be added already exists!") { } + KEY_IMAGE_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +/*********************************** + * End of Exception Definitions + ***********************************/ + + +class BlockchainDB +{ +private: + /********************************************************************* + * private virtual members + *********************************************************************/ + + // tells the subclass to add the block and metadata to storage + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) = 0; + + // tells the subclass to remove data about the top block + virtual void remove_block() = 0; + + // tells the subclass to store the transaction and its metadata + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + + // tells the subclass to remove data about a transaction + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; + + // tells the subclass to store an output + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0; + + // tells the subclass to remove an output + virtual void remove_output(const tx_out& tx_output) = 0; + + // tells the subclass to store a spent key + virtual void add_spent_key(const crypto::key_image& k_image) = 0; + + // tells the subclass to remove a spent key + virtual void remove_spent_key(const crypto::key_image& k_image) = 0; + + + /********************************************************************* + * private concrete members + *********************************************************************/ + // private version of pop_block, for undoing if an add_block goes tits up + void pop_block(); + + // helper function for add_transactions, to add each individual tx + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); + + // helper function to remove transaction from blockchain + void remove_transaction(const crypto::hash& tx_hash); + + uint64_t num_calls = 0; + uint64_t time_blk_hash = 0; + uint64_t time_add_block1 = 0; + uint64_t time_add_transaction = 0; + + +protected: + + mutable uint64_t time_tx_exists = 0; + uint64_t time_commit1 = 0; + + +public: + + // virtual dtor + virtual ~BlockchainDB() { }; + + // reset profiling stats + void reset_stats(); + + // show profiling stats + void show_stats(); + + // open the db at location <filename>, or create it if there isn't one. + virtual void open(const std::string& filename, const int db_flags = 0) = 0; + + // returns true of the db is open/ready, else false + bool is_open(); + + // close and sync the db + virtual void close() = 0; + + // sync the db + virtual void sync() = 0; + + // reset the db -- USE WITH CARE + virtual void reset() = 0; + + // get all files used by this db (if any) + virtual std::vector<std::string> get_filenames() const = 0; + + // return the name of the folder the db's file(s) should reside in + virtual std::string get_db_name() const = 0; + + + // FIXME: these are just for functionality mocking, need to implement + // RAII-friendly and multi-read one-write friendly locking mechanism + // + // acquire db lock + virtual bool lock() = 0; + + // release db lock + virtual void unlock() = 0; + + virtual void batch_start() = 0; + virtual void batch_stop() = 0; + virtual void set_batch_transactions(bool) = 0; + + // adds a block with the given metadata to the top of the blockchain, returns the new height + // NOTE: subclass implementations of this (or the functions it calls) need + // to handle undoing any partially-added blocks in the event of a failure. + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ); + + // return true if a block with hash <h> exists in the blockchain + virtual bool block_exists(const crypto::hash& h) const = 0; + + // return block with hash <h> + virtual block get_block(const crypto::hash& h) const = 0; + + // return the height of the block with hash <h> on the blockchain, + // throw if it doesn't exist + virtual uint64_t get_block_height(const crypto::hash& h) const = 0; + + // return header for block with hash <h> + virtual block_header get_block_header(const crypto::hash& h) const = 0; + + // return block at height <height> + virtual block get_block_from_height(const uint64_t& height) const = 0; + + // return timestamp of block at height <height> + virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; + + // return timestamp of most recent block + virtual uint64_t get_top_block_timestamp() const = 0; + + // return block size of block at height <height> + virtual size_t get_block_size(const uint64_t& height) const = 0; + + // return cumulative difficulty up to and including block at height <height> + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const = 0; + + // return difficulty of block at height <height> + virtual difficulty_type get_block_difficulty(const uint64_t& height) const = 0; + + // return number of coins generated up to and including block at height <height> + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; + + // return hash of block at height <height> + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const = 0; + + // return vector of blocks in range <h1,h2> of height (inclusively) + virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const = 0; + + // return vector of block hashes in range <h1, h2> of height (inclusively) + virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const = 0; + + // return the hash of the top block on the chain + virtual crypto::hash top_block_hash() const = 0; + + // return the block at the top of the blockchain + virtual block get_top_block() const = 0; + + // return the height of the chain + virtual uint64_t height() const = 0; + + // pops the top block off the blockchain. + // Returns by reference the popped block and its associated transactions + // + // IMPORTANT: + // When a block is popped, the transactions associated with it need to be + // removed, as well as outputs and spent key images associated with + // those transactions. + virtual void pop_block(block& blk, std::vector<transaction>& txs); + + + // return true if a transaction with hash <h> exists + virtual bool tx_exists(const crypto::hash& h) const = 0; + + // return unlock time of tx with hash <h> + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; + + // return tx with hash <h> + // throw if no such tx exists + virtual transaction get_tx(const crypto::hash& h) const = 0; + + // returns the total number of transactions in all blocks + virtual uint64_t get_tx_count() const = 0; + + // return list of tx with hashes <hlist>. + // TODO: decide if a missing hash means return empty list + // or just skip that hash + virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const = 0; + + // returns height of block that contains transaction with hash <h> + virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0; + + // return global output index of a random output of amount <amount> + virtual uint64_t get_random_output(const uint64_t& amount) const = 0; + + // returns the total number of outputs of amount <amount> + virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0; + + // return public key for output with global output amount <amount> and index <index> + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; + + // returns the output indexed by <index> in the transaction with hash <h> + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0; + + // returns the tx hash associated with an output, referenced by global output index + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const = 0; + + // returns the transaction-local reference for the output with <amount> at <index> + // return type is pair of tx hash and index + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0; + + // return a vector of indices corresponding to the global output index for + // each output in the transaction with hash <h> + virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const = 0; + // return a vector of indices corresponding to the amount output index for + // each output in the transaction with hash <h> + virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const = 0; + + // returns true if key image <img> is present in spent key images storage + virtual bool has_key_image(const crypto::key_image& img) const = 0; + + bool m_open; +}; // class BlockchainDB + + +} // namespace cryptonote + +#endif // BLOCKCHAIN_DB_H diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h new file mode 100644 index 000000000..b13007df4 --- /dev/null +++ b/src/blockchain_db/db_types.h @@ -0,0 +1,40 @@ +// Copyright (c) 2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#pragma once + +namespace cryptonote +{ + + const std::unordered_set<std::string> blockchain_db_types = + { "lmdb" + , "berkeley" + }; + +} // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp new file mode 100644 index 000000000..8e09dfab2 --- /dev/null +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -0,0 +1,1814 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_lmdb.h" + +#include <boost/filesystem.hpp> +#include <memory> // std::unique_ptr +#include <cstring> // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace +{ + +template <typename T> +inline void throw0(const T &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +template <typename T> +inline void throw1(const T &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct lmdb_cur +{ + lmdb_cur(MDB_txn* txn, MDB_dbi dbi) + { + if (mdb_cursor_open(txn, dbi, &m_cur)) + throw0(cryptonote::DB_ERROR("Error opening db cursor")); + done = false; + } + + ~lmdb_cur() { close(); } + + operator MDB_cursor*() { return m_cur; } + operator MDB_cursor**() { return &m_cur; } + + void close() + { + if (!done) + { + mdb_cursor_close(m_cur); + done = true; + } + } + +private: + MDB_cursor* m_cur; + bool done; +}; + +template<typename T> +struct MDB_val_copy: public MDB_val +{ + MDB_val_copy(const T &t): t_copy(t) + { + mv_size = sizeof (T); + mv_data = &t_copy; + } +private: + T t_copy; +}; + +template<> +struct MDB_val_copy<cryptonote::blobdata>: public MDB_val +{ + MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()]) + { + memcpy(data.get(), bd.data(), bd.size()); + mv_size = bd.size(); + mv_data = data.get(); + } +private: + std::unique_ptr<char[]> data; +}; + +auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { + const uint64_t va = *(const uint64_t*)a->mv_data; + const uint64_t vb = *(const uint64_t*)b->mv_data; + if (va < vb) return -1; + else if (va == vb) return 0; + else return 1; +}; + +const char* const LMDB_BLOCKS = "blocks"; +const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; +const char* const LMDB_BLOCK_HASHES = "block_hashes"; +const char* const LMDB_BLOCK_SIZES = "block_sizes"; +const char* const LMDB_BLOCK_DIFFS = "block_diffs"; +const char* const LMDB_BLOCK_COINS = "block_coins"; + +const char* const LMDB_TXS = "txs"; +const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; +const char* const LMDB_TX_HEIGHTS = "tx_heights"; +const char* const LMDB_TX_OUTPUTS = "tx_outputs"; + +const char* const LMDB_OUTPUT_TXS = "output_txs"; +const char* const LMDB_OUTPUT_INDICES = "output_indices"; +const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const LMDB_OUTPUT_KEYS = "output_keys"; +const char* const LMDB_OUTPUTS = "outputs"; +const char* const LMDB_OUTPUT_GINDICES = "output_gindices"; +const char* const LMDB_SPENT_KEYS = "spent_keys"; + +inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) +{ + if (mdb_dbi_open(txn, name, flags, &dbi)) + throw0(cryptonote::DB_OPEN_FAILURE(error_string.c_str())); +} + +} // anonymous namespace + +namespace cryptonote +{ + +// If m_batch_active is set, a batch transaction exists beyond this class, such +// as a batch import with verification enabled, or possibly (later) a batch +// network sync. +// +// For some of the lookup methods, such as get_block_timestamp(), tx_exists(), +// and get_tx(), when m_batch_active is set, the lookup uses the batch +// transaction. This isn't only because the transaction is available, but it's +// necessary so that lookups include the database updates only present in the +// current batch write. +// +// A regular network sync without batch writes is expected to open a new read +// transaction, as those lookups are part of the validation done prior to the +// write for block and tx data, so no write transaction is open at the time. + +void BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::hash> val_h(blk_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); + + if (m_height > 0) + { + MDB_val_copy<crypto::hash> parent_key(blk.prev_id); + MDB_val parent_h; + if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + } + uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; + if (parent_height != m_height - 1) + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); + } + + MDB_val_copy<uint64_t> key(m_height); + + MDB_val_copy<blobdata> blob(block_to_blob(blk)); + auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str())); + + MDB_val_copy<size_t> sz(block_size); + if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) + throw0(DB_ERROR("Failed to add block size to db transaction")); + + MDB_val_copy<uint64_t> ts(blk.timestamp); + if (mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction")); + + MDB_val_copy<difficulty_type> diff(cumulative_difficulty); + if (mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); + + MDB_val_copy<uint64_t> coinsgen(coins_generated); + if (mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); + + if (mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction")); + + if (mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0)) + throw0(DB_ERROR("Failed to add block hash to db transaction")); + +} + +void BlockchainLMDB::remove_block() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + + MDB_val_copy<uint64_t> k(m_height - 1); + MDB_val h; + if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + + if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block to db transaction")); + + if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + + if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + + if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + + if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + + if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + + if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); +} + +void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::hash> val_h(tx_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + + MDB_val_copy<blobdata> blob(tx_to_blob(tx)); + if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) + throw0(DB_ERROR("Failed to add tx blob to db transaction")); + + MDB_val_copy<uint64_t> height(m_height); + if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) + throw0(DB_ERROR("Failed to add tx block height to db transaction")); + + MDB_val_copy<uint64_t> unlock_time(tx.unlock_time); + if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0)) + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); +} + +void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::hash> val_h(tx_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + + if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); + if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + + remove_tx_outputs(tx_hash, tx); + + if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); + +} + +void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<uint64_t> k(m_num_outputs); + MDB_val_copy<crypto::hash> v(tx_hash); + + if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); + if (mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + MDB_val_copy<uint64_t> val_local_index(local_index); + if (mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + MDB_val_copy<uint64_t> val_amount(tx_output.amount); + if (auto result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0)) + throw0(DB_ERROR(std::string("Failed to add output amount to db transaction").append(mdb_strerror(result)).c_str())); + + if (tx_output.target.type() == typeid(txout_to_key)) + { + MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); + if (mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + } + + +/****** Uncomment if ever outputs actually need to be stored in this manner + * + blobdata b = output_to_blob(tx_output); + + v.mv_size = b.size(); + v.mv_data = &b; + if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output to db transaction")); + if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0)) + throw0(DB_ERROR("Failed to add output global index to db transaction")); +************************************************************************/ + + m_num_outputs++; +} + +void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + lmdb_cur cur(*m_write_txn, m_tx_outputs); + + MDB_val_copy<crypto::hash> k(tx_hash); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + LOG_ERROR("Attempting to remove a tx's outputs, but none found. Continuing, but...be wary, because that's weird."); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + else + { + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + const tx_out tx_output = tx.vout[i]; + remove_output(*(const uint64_t*)v.mv_data, tx_output.amount); + if (i < num_elems - 1) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + } + } + + cur.close(); +} + +// TODO: probably remove this function +void BlockchainLMDB::remove_output(const tx_out& tx_output) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); + return; +} + +void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amount) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<uint64_t> k(out_index); + +/****** Uncomment if ever outputs actually need to be stored in this manner + blobdata b; + t_serializable_object_to_blob(tx_output, b); + k.mv_size = b.size(); + k.mv_data = &b; + + if (mdb_get(*m_write_txn, m_output_gindices, &k, &v)) + throw1(OUTPUT_DNE("Attempting to remove output that does not exist")); + + uint64_t gindex = *(uint64_t*)v.mv_data; + + auto result = mdb_del(*m_write_txn, m_output_gindices, &k, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of output global index to db transaction")); + + result = mdb_del(*m_write_txn, m_outputs, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of output to db transaction")); +*********************************************************************/ + + auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } + + result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); + // if (result != 0 && result != MDB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } + + result = mdb_del(*m_write_txn, m_output_keys, &k, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + } + else if (result) + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + + remove_amount_output_index(amount, out_index); + + m_num_outputs--; +} + +void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + lmdb_cur cur(*m_write_txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + goi = *(const uint64_t *)v.mv_data; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = mdb_cursor_del(cur, 0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str())); + } + else + { + // not found + cur.close(); + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); +} + +void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::key_image> val_key(k_image); + MDB_val unused; + if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); + + char anything = '\0'; + unused.mv_size = sizeof(char); + unused.mv_data = &anything; + if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); +} + +void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::key_image> k(k_image); + auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of key image to db transaction")); +} + +blobdata BlockchainLMDB::output_to_blob(const tx_out& output) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + throw1(DB_ERROR("Error serializing output to blob")); + return b; +} + +tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + std::stringstream ss; + ss << blob; + binary_archive<false> ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + throw1(DB_ERROR("Error deserializing tx output blob")); + + return o; +} + +uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(const uint64_t*)v.mv_data; + + cur.close(); + + txn.commit(); + + return glob_index; +} + +void BlockchainLMDB::check_open() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + +BlockchainLMDB::~BlockchainLMDB() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + // batch transaction shouldn't be active at this point. If it is, consider it aborted. + if (m_batch_active) + batch_abort(); +} + +BlockchainLMDB::BlockchainLMDB(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_batch_active = false; + m_height = 0; +} + +void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + if (m_open) + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed")); + } + else + { + if (!boost::filesystem::create_directory(direc)) + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); + } + + // check for existing LMDB files in base directory + boost::filesystem::path old_files = direc.parent_path(); + if (boost::filesystem::exists(old_files / "data.mdb") || + boost::filesystem::exists(old_files / "lock.mdb")) + { + LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str()); + LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); + throw DB_ERROR("Database could not be opened"); + } + + m_folder = filename; + + // set up lmdb environment + if (mdb_env_create(&m_env)) + throw0(DB_ERROR("Failed to create lmdb environment")); + if (mdb_env_set_maxdbs(m_env, 20)) + throw0(DB_ERROR("Failed to set max number of dbs")); + + size_t mapsize = 1LL << 34; + if (auto result = mdb_env_set_mapsize(m_env, mapsize)) + throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); + if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) + throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); + + // get a read/write MDB_txn + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + // open necessary databases, and set properties as needed + // uses macros to avoid having to change things too many places + lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); + + lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); + lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); + lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); + lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); + lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); + + lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); + lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); + lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); + +/*************** not used, but kept for posterity + lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs"); + lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); +*************************************************/ + + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); + + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); + + // get and keep current height + MDB_stat db_stats; + if (mdb_stat(txn, m_blocks, &db_stats)) + throw0(DB_ERROR("Failed to query m_blocks")); + LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); + m_height = db_stats.ms_entries; + + // get and keep current number of outputs + if (mdb_stat(txn, m_output_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_output_indices")); + m_num_outputs = db_stats.ms_entries; + + // commit the transaction + txn.commit(); + + m_open = true; + // from here, init should be finished +} + +void BlockchainLMDB::close() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (m_batch_active) + { + LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); + batch_abort(); + } + this->sync(); + + // FIXME: not yet thread safe!!! Use with care. + mdb_env_close(m_env); +} + +void BlockchainLMDB::sync() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part + // MDB_NOMETASYNC. Force flush to be synchronous. + if (auto result = mdb_env_sync(m_env, true)) + { + throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str())); + } +} + +void BlockchainLMDB::reset() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // TODO: this +} + +std::vector<std::string> BlockchainLMDB::get_filenames() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + std::vector<std::string> filenames; + + boost::filesystem::path datafile(m_folder); + datafile /= "data.mdb"; + boost::filesystem::path lockfile(m_folder); + lockfile /= "lock.mdb"; + + filenames.push_back(datafile.string()); + filenames.push_back(lockfile.string()); + + return filenames; +} + +std::string BlockchainLMDB::get_db_name() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + return std::string("lmdb"); +} + +// TODO: this? +bool BlockchainLMDB::lock() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + return false; +} + +// TODO: this? +void BlockchainLMDB::unlock() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); +} + +bool BlockchainLMDB::block_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_block_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + + txn.commit(); + return true; +} + +block BlockchainLMDB::get_block(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + return get_block_from_height(get_block_height(h)); +} + +uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_block_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // block_header object is automatically cast from block object + return get_block(h); +} + +block BlockchainLMDB::get_block_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(txn, m_blocks, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + return b; +} + +uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +uint64_t BlockchainLMDB::get_top_block_timestamp() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); +} + +size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const size_t*)result.mv_data; +} + +difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + if (! m_batch_active) + txn.commit(); + return *(difficulty_type*)result.mv_data; +} + +difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; +} + +uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); + + if (! m_batch_active) + txn.commit(); + return *(crypto::hash*)result.mv_data; +} + +std::vector<block> BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<block> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + + return v; +} + +std::vector<crypto::hash> BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<crypto::hash> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + + return v; +} + +crypto::hash BlockchainLMDB::top_block_hash() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + if (m_height != 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; +} + +block BlockchainLMDB::get_top_block() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + if (m_height != 0) + { + return get_block_from_height(m_height - 1); + } + + block b; + return b; +} + +uint64_t BlockchainLMDB::height() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + return m_height; +} + +bool BlockchainLMDB::tx_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + + TIME_MEASURE_START(time1); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (get_result == MDB_NOTFOUND) + { + if (! m_batch_active) + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + + return true; +} + +uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + + return *(const uint64_t*)result.mv_data; +} + +transaction BlockchainLMDB::get_tx(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (! m_batch_active) + txn.commit(); + + return tx; +} + +uint64_t BlockchainLMDB::get_tx_count() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_stat db_stats; + if (mdb_stat(txn, m_txs, &db_stats)) + throw0(DB_ERROR("Failed to query m_txs")); + + txn.commit(); + + return db_stats.ms_entries; +} + +std::vector<transaction> BlockchainLMDB::get_tx_list(const std::vector<crypto::hash>& hlist) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<transaction> v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + + return v; +} + +uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + + if (! m_batch_active) + txn.commit(); + + return *(const uint64_t*)result.mv_data; +} + +//FIXME: make sure the random method used here is appropriate +uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); + + return crypto::rand<uint64_t>() % num_outputs; +} + +uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + return 0; + } + else if (result) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + txn.commit(); + + return num_elems; +} + +crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<uint64_t> k(glob_index); + MDB_val v; + auto get_result = mdb_get(txn, m_output_keys, &k, &v); + if (get_result == MDB_NOTFOUND) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + return *(crypto::public_key*)v.mv_data; +} + +tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val_copy<crypto::hash> k(h); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + blobdata b; + b = *(blobdata*)v.mv_data; + + cur.close(); + txn.commit(); + + return output_from_blob(b); +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainLMDB::get_output(const uint64_t& index) const +{ + return tx_out(); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<uint64_t> k(index); + MDB_val v; + auto get_result = mdb_get(txn, m_outputs, &k, &v); + if (get_result == MDB_NOTFOUND) + { + throw OUTPUT_DNE("Attempting to get output by global index, but output does not exist"); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output from the db")); + + blobdata b = *(blobdata*)v.mv_data; + + return output_from_blob(b); +} + +tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy<uint64_t> k(index); + MDB_val v; + + auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = *(crypto::hash*)v.mv_data; + + get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + if (! m_batch_active) + txn.commit(); + + return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); +} + +tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + lmdb_cur cur(*txn_ptr, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(const uint64_t*)v.mv_data; + + cur.close(); + + if (! m_batch_active) + txn.commit(); + + return get_output_tx_and_index_from_global(glob_index); +} + +std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val_copy<crypto::hash> k(h); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + index_vec.push_back(*(const uint64_t *)v.mv_data); + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + cur.close(); + txn.commit(); + + return index_vec; +} + +std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + std::vector<uint64_t> index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + BOOST_FOREACH(const auto& vout, tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + output_index = *(const uint64_t *)v.mv_data; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; +} + + + +bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::key_image> val_key(img); + MDB_val unused; + if (mdb_get(txn, m_spent_keys, &val_key, &unused) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); + return false; +} + +void BlockchainLMDB::batch_start() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (m_batch_active) + throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_txn) + throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); + check_open(); + // NOTE: need to make sure it's destroyed properly when done + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + // indicates this transaction is for batch transactions, but not whether it's + // active + m_write_batch_txn.m_batch_txn = true; + m_write_txn = &m_write_batch_txn; + m_batch_active = true; + LOG_PRINT_L3("batch transaction: begin"); +} + +void BlockchainLMDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + LOG_PRINT_L3("batch transaction: committed"); + + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + if (! m_write_batch_txn.m_batch_txn) + throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); + m_write_txn = &m_write_batch_txn; +} + +void BlockchainLMDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + // for destruction of batch transaction + m_write_txn = nullptr; + m_batch_active = false; + LOG_PRINT_L3("batch transaction: end"); +} + +void BlockchainLMDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + // for destruction of batch transaction + m_write_txn = nullptr; + // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. + m_write_batch_txn.abort(); + m_batch_active = false; + LOG_PRINT_L3("batch transaction: aborted"); +} + +void BlockchainLMDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + +uint64_t BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + if (! m_batch_active) + { + m_write_txn = NULL; + + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } + } + catch (...) + { + m_num_outputs = num_outputs; + if (! m_batch_active) + m_write_txn = NULL; + throw; + } + + return ++m_height; +} + +void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + if (! m_batch_active) + { + m_write_txn = NULL; + + txn.commit(); + } + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + --m_height; +} + +} // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h new file mode 100644 index 000000000..8f1e07e0d --- /dev/null +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -0,0 +1,314 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata + +#include <lmdb.h> + +namespace cryptonote +{ + +struct mdb_txn_safe +{ + mdb_txn_safe() : m_txn(NULL) { } + ~mdb_txn_safe() + { + LOG_PRINT_L3("mdb_txn_safe: destructor"); + if (m_txn != NULL) + { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } + mdb_txn_abort(m_txn); + } + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (mdb_txn_commit(m_txn)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + // This should only be needed for batch transaction which must be ensured to + // be aborted before mdb_env_close, not after. So we can't rely on + // BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late + // to properly abort, since mdb_env_close would have been called earlier. + void abort() + { + LOG_PRINT_L3("mdb_txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); + } + } + + operator MDB_txn*() + { + return m_txn; + } + + operator MDB_txn**() + { + return &m_txn; + } + + MDB_txn* m_txn; + bool m_batch_txn = false; +}; + + +class BlockchainLMDB : public BlockchainDB +{ +public: + BlockchainLMDB(bool batch_transactions=false); + ~BlockchainLMDB(); + + virtual void open(const std::string& filename, const int mdb_flags=0); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector<std::string> get_filenames() const; + + virtual std::string get_db_name() const; + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h) const; + + virtual block get_block(const crypto::hash& h) const; + + virtual uint64_t get_block_height(const crypto::hash& h) const; + + virtual block_header get_block_header(const crypto::hash& h) const; + + virtual block get_block_from_height(const uint64_t& height) const; + + virtual uint64_t get_block_timestamp(const uint64_t& height) const; + + virtual uint64_t get_top_block_timestamp() const; + + virtual size_t get_block_size(const uint64_t& height) const; + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; + + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; + + virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual crypto::hash top_block_hash() const; + + virtual block get_top_block() const; + + virtual uint64_t height() const; + + virtual bool tx_exists(const crypto::hash& h) const; + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; + + virtual transaction get_tx(const crypto::hash& h) const; + + virtual uint64_t get_tx_count() const; + + virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const; + + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; + + virtual uint64_t get_random_output(const uint64_t& amount) const; + + virtual uint64_t get_num_outputs(const uint64_t& amount) const; + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; + + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; + + virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; + + virtual bool has_key_image(const crypto::key_image& img) const; + + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ); + + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + + virtual void pop_block(block& blk, std::vector<transaction>& txs); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& block_hash + ); + + virtual void remove_block(); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob) const; + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + + void check_open() const; + + MDB_env* m_env; + + MDB_dbi m_blocks; + MDB_dbi m_block_heights; + MDB_dbi m_block_hashes; + MDB_dbi m_block_timestamps; + MDB_dbi m_block_sizes; + MDB_dbi m_block_diffs; + MDB_dbi m_block_coins; + + MDB_dbi m_txs; + MDB_dbi m_tx_unlocks; + MDB_dbi m_tx_heights; + MDB_dbi m_tx_outputs; + + MDB_dbi m_output_txs; + MDB_dbi m_output_indices; + MDB_dbi m_output_gindices; + MDB_dbi m_output_amounts; + MDB_dbi m_output_keys; + MDB_dbi m_outputs; + + MDB_dbi m_spent_keys; + + uint64_t m_height; + uint64_t m_num_outputs; + std::string m_folder; + mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress +}; + +} // namespace cryptonote diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 739c0adee..620e1add9 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -39,8 +39,11 @@ set(common_private_headers boost_serialization_helper.h command_line.h dns_utils.h + http_connection.h int-util.h pod-class.h + rpc_client.h + scoped_message_writer.h unordered_containers_boost_serialization.h util.h varint.h) diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 9ea6ba1ba..6adc46ca7 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/base58.h b/src/common/base58.h index ad4f146a4..1a9ad1a77 100644 --- a/src/common/base58.h +++ b/src/common/base58.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/boost_serialization_helper.h b/src/common/boost_serialization_helper.h index 7c5529cc2..c108c52e1 100644 --- a/src/common/boost_serialization_helper.h +++ b/src/common/boost_serialization_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index a2e7faf9f..d2cd75e5b 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -48,4 +48,7 @@ namespace command_line const arg_descriptor<bool> arg_version = {"version", "Output version information"}; const arg_descriptor<std::string> arg_data_dir = {"data-dir", "Specify data directory"}; const arg_descriptor<std::string> arg_testnet_data_dir = {"testnet-data-dir", "Specify testnet data directory"}; + const arg_descriptor<bool> arg_test_drop_download = {"test-drop-download", "For net tests: in download, discard ALL blocks instead checking/saving them (very fast)"}; + const arg_descriptor<uint64_t> arg_test_drop_download_height = {"test-drop-download-height", "Like test-drop-download but disards only after around certain height", 0}; + const arg_descriptor<int> arg_test_dbg_lock_sleep = {"test-dbg-lock-sleep", "Sleep time in ms, defaults to 0 (off), used to debug before/after locking mutex. Values 100 to 1000 are good for tests."}; } diff --git a/src/common/command_line.h b/src/common/command_line.h index 09ae877d3..ae79f0a05 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -204,4 +204,7 @@ namespace command_line extern const arg_descriptor<bool> arg_version; extern const arg_descriptor<std::string> arg_data_dir; extern const arg_descriptor<std::string> arg_testnet_data_dir; + extern const arg_descriptor<bool> arg_test_drop_download; + extern const arg_descriptor<uint64_t> arg_test_drop_download_height; + extern const arg_descriptor<int> arg_test_dbg_lock_sleep; } diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index d21b538a6..ea7f1078b 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -34,7 +34,65 @@ #include <stdlib.h> #include "include_base_utils.h" +#include <boost/filesystem/fstream.hpp> using namespace epee; +namespace bf = boost::filesystem; + +namespace +{ + +/* + * The following two functions were taken from unbound-anchor.c, from + * the unbound library packaged with this source. The license and source + * can be found in $PROJECT_ROOT/external/unbound + */ + +/* Cert builtin commented out until it's used, as the compiler complains + +// return the built in root update certificate +static const char* +get_builtin_cert(void) +{ + return +// The ICANN CA fetched at 24 Sep 2010. Valid to 2028 +"-----BEGIN CERTIFICATE-----\n" +"MIIDdzCCAl+gAwIBAgIBATANBgkqhkiG9w0BAQsFADBdMQ4wDAYDVQQKEwVJQ0FO\n" +"TjEmMCQGA1UECxMdSUNBTk4gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxFjAUBgNV\n" +"BAMTDUlDQU5OIFJvb3QgQ0ExCzAJBgNVBAYTAlVTMB4XDTA5MTIyMzA0MTkxMloX\n" +"DTI5MTIxODA0MTkxMlowXTEOMAwGA1UEChMFSUNBTk4xJjAkBgNVBAsTHUlDQU5O\n" +"IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRYwFAYDVQQDEw1JQ0FOTiBSb290IENB\n" +"MQswCQYDVQQGEwJVUzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKDb\n" +"cLhPNNqc1NB+u+oVvOnJESofYS9qub0/PXagmgr37pNublVThIzyLPGCJ8gPms9S\n" +"G1TaKNIsMI7d+5IgMy3WyPEOECGIcfqEIktdR1YWfJufXcMReZwU4v/AdKzdOdfg\n" +"ONiwc6r70duEr1IiqPbVm5T05l1e6D+HkAvHGnf1LtOPGs4CHQdpIUcy2kauAEy2\n" +"paKcOcHASvbTHK7TbbvHGPB+7faAztABLoneErruEcumetcNfPMIjXKdv1V1E3C7\n" +"MSJKy+jAqqQJqjZoQGB0necZgUMiUv7JK1IPQRM2CXJllcyJrm9WFxY0c1KjBO29\n" +"iIKK69fcglKcBuFShUECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8B\n" +"Af8EBAMCAf4wHQYDVR0OBBYEFLpS6UmDJIZSL8eZzfyNa2kITcBQMA0GCSqGSIb3\n" +"DQEBCwUAA4IBAQAP8emCogqHny2UYFqywEuhLys7R9UKmYY4suzGO4nkbgfPFMfH\n" +"6M+Zj6owwxlwueZt1j/IaCayoKU3QsrYYoDRolpILh+FPwx7wseUEV8ZKpWsoDoD\n" +"2JFbLg2cfB8u/OlE4RYmcxxFSmXBg0yQ8/IoQt/bxOcEEhhiQ168H2yE5rxJMt9h\n" +"15nu5JBSewrCkYqYYmaxyOC3WrVGfHZxVI7MpIFcGdvSb2a1uyuua8l0BKgk3ujF\n" +"0/wsHNeP22qNyVO+XVBzrM8fk8BSUFuiT/6tZTYXRtEt5aKQZgXbKU5dUF3jT9qg\n" +"j/Br5BZw3X/zd325TvnswzMC1+ljLzHnQGGk\n" +"-----END CERTIFICATE-----\n" + ; +} +*/ + +/** return the built in root DS trust anchor */ +static const char* +get_builtin_ds(void) +{ + return +". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n"; +} + +/************************************************************ + ************************************************************ + ***********************************************************/ + +} // anonymous namespace namespace tools { @@ -104,11 +162,28 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) // init libunbound context m_data->m_ub_context = ub_ctx_create(); - char empty_string = '\0'; - // look for "/etc/resolv.conf" and "/etc/hosts" or platform equivalent - ub_ctx_resolvconf(m_data->m_ub_context, &empty_string); - ub_ctx_hosts(m_data->m_ub_context, &empty_string); + ub_ctx_resolvconf(m_data->m_ub_context, NULL); + ub_ctx_hosts(m_data->m_ub_context, NULL); + + #ifdef DEVELOPER_LIBUNBOUND_OLD + #pragma message "Using the work around for old libunbound" + { // work around for bug https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=515 needed for it to compile on e.g. Debian 7 + char * ds_copy = NULL; // this will be the writable copy of string that bugged version of libunbound requires + try { + char * ds_copy = strdup( ::get_builtin_ds() ); + ub_ctx_add_ta(m_data->m_ub_context, ds_copy); + } catch(...) { // probably not needed but to work correctly in every case... + if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup + throw ; + } + if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup + } + #else + // normal version for fixed libunbound + ub_ctx_add_ta(m_data->m_ub_context, ::get_builtin_ds() ); + #endif + } DNSResolver::~DNSResolver() @@ -143,6 +218,8 @@ std::vector<std::string> DNSResolver::get_ipv4(const std::string& url, bool& dns // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_A, DNS_CLASS_IN, &(result.ptr))) { + dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus)); + dnssec_valid = !result.ptr->bogus; if (result.ptr->havedata) { for (size_t i=0; result.ptr->data[i] != NULL; i++) @@ -175,6 +252,8 @@ std::vector<std::string> DNSResolver::get_ipv6(const std::string& url, bool& dns // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_AAAA, DNS_CLASS_IN, &(result.ptr))) { + dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus)); + dnssec_valid = !result.ptr->bogus; if (result.ptr->havedata) { for (size_t i=0; result.ptr->data[i] != NULL; i++) @@ -207,6 +286,8 @@ std::vector<std::string> DNSResolver::get_txt_record(const std::string& url, boo // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_TXT, DNS_CLASS_IN, &(result.ptr))) { + dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus)); + dnssec_valid = !result.ptr->bogus; if (result.ptr->havedata) { for (size_t i=0; result.ptr->data[i] != NULL; i++) diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index 9958e4ca2..a16c7eff7 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/http_connection.h b/src/common/http_connection.h new file mode 100644 index 000000000..759145009 --- /dev/null +++ b/src/common/http_connection.h @@ -0,0 +1,42 @@ +#pragma once + +#include "string_tools.h" +#include "net/http_client.h" + +namespace tools { + +class t_http_connection { +private: + epee::net_utils::http::http_simple_client * mp_http_client; + bool m_ok; +public: + static unsigned int const TIMEOUT = 200000; + + t_http_connection( + epee::net_utils::http::http_simple_client * p_http_client + , uint32_t ip + , uint16_t port + ) + : mp_http_client(p_http_client) + { + // TODO fix http client so that it accepts properly typed arguments + std::string ip_str = epee::string_tools::get_ip_string_from_int32(ip); + std::string port_str = boost::lexical_cast<std::string>(port); + m_ok = mp_http_client->connect(ip_str, port_str, TIMEOUT); + } + + ~t_http_connection() + { + if (m_ok) + { + mp_http_client->disconnect(); + } + } + + bool is_open() + { + return m_ok; + } +}; // class t_http_connection + +} // namespace tools diff --git a/src/common/int-util.h b/src/common/int-util.h index 40ac2eb64..cd6e65460 100644 --- a/src/common/int-util.h +++ b/src/common/int-util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/pod-class.h b/src/common/pod-class.h index 8c7e4c2e9..ebe919203 100644 --- a/src/common/pod-class.h +++ b/src/common/pod-class.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h new file mode 100644 index 000000000..a6d4b6cc1 --- /dev/null +++ b/src/common/rpc_client.h @@ -0,0 +1,132 @@ +#pragma once + +#include "common/http_connection.h" +#include "common/scoped_message_writer.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "storages/http_abstract_invoke.h" +#include "net/http_client.h" +#include "string_tools.h" +#include <boost/lexical_cast.hpp> + +namespace tools +{ + class t_rpc_client final + { + private: + epee::net_utils::http::http_simple_client m_http_client; + uint32_t m_ip; + uint16_t m_port; + public: + t_rpc_client( + uint32_t ip + , uint16_t port + ) + : m_http_client{} + , m_ip{ip} + , m_port{port} + {} + + std::string build_url(std::string const & relative_url) + { + std::string result = + "http://" + + epee::string_tools::get_ip_string_from_int32(m_ip) + + ":" + + boost::lexical_cast<std::string>(m_port) + + relative_url; + return result; + } + + template <typename T_req, typename T_res> + bool basic_json_rpc_request( + T_req & req + , T_res & res + , std::string const & method_name + ) + { + std::string rpc_url = build_url("/json_rpc"); + t_http_connection connection(&m_http_client, m_ip, m_port); + + bool ok = connection.is_open(); + if (!ok) + { + fail_msg_writer() << "Couldn't connect to daemon"; + return false; + } + ok = ok && epee::net_utils::invoke_http_json_rpc(rpc_url, method_name, req, res, m_http_client); + if (!ok) + { + fail_msg_writer() << "Daemon request failed"; + return false; + } + else + { + return true; + } + } + + template <typename T_req, typename T_res> + bool json_rpc_request( + T_req & req + , T_res & res + , std::string const & method_name + , std::string const & fail_msg + ) + { + std::string rpc_url = build_url("/json_rpc"); + t_http_connection connection(&m_http_client, m_ip, m_port); + + bool ok = connection.is_open(); + ok = ok && epee::net_utils::invoke_http_json_rpc(rpc_url, method_name, req, res, m_http_client); + if (!ok) + { + fail_msg_writer() << "Couldn't connect to daemon"; + return false; + } + else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? + { + fail_msg_writer() << fail_msg << " -- " << res.status; + return false; + } + else + { + return true; + } + } + + template <typename T_req, typename T_res> + bool rpc_request( + T_req & req + , T_res & res + , std::string const & relative_url + , std::string const & fail_msg + ) + { + std::string rpc_url = build_url(relative_url); + t_http_connection connection(&m_http_client, m_ip, m_port); + + bool ok = connection.is_open(); + ok = ok && epee::net_utils::invoke_http_json_remote_command2(rpc_url, req, res, m_http_client); + if (!ok) + { + fail_msg_writer() << "Couldn't connect to daemon"; + return false; + } + else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? + { + fail_msg_writer() << fail_msg << " -- " << res.status; + return false; + } + else + { + return true; + } + } + + bool check_connection() + { + t_http_connection connection(&m_http_client, m_ip, m_port); + return connection.is_open(); + } + }; +} diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h new file mode 100644 index 000000000..70db54eae --- /dev/null +++ b/src/common/scoped_message_writer.h @@ -0,0 +1,95 @@ +#pragma once + +#include "misc_log_ex.h" +#include <iostream> + +namespace tools +{ + +class scoped_message_writer +{ +private: + bool m_flush; + std::stringstream m_oss; + epee::log_space::console_colors m_color; + bool m_bright; + int m_log_level; +public: + scoped_message_writer( + epee::log_space::console_colors color = epee::log_space::console_color_default + , bool bright = false + , std::string&& prefix = std::string() + , int log_level = LOG_LEVEL_2 + ) + : m_flush(true) + , m_color(color) + , m_bright(bright) + , m_log_level(log_level) + { + m_oss << prefix; + } + + scoped_message_writer(scoped_message_writer&& rhs) + : m_flush(std::move(rhs.m_flush)) +#if defined(_MSC_VER) + , m_oss(std::move(rhs.m_oss)) +#else + // GCC bug: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 + , m_oss(rhs.m_oss.str(), std::ios_base::out | std::ios_base::ate) +#endif + , m_color(std::move(rhs.m_color)) + , m_log_level(std::move(rhs.m_log_level)) + { + rhs.m_flush = false; + } + + scoped_message_writer(scoped_message_writer& rhs) = delete; + scoped_message_writer& operator=(scoped_message_writer& rhs) = delete; + scoped_message_writer& operator=(scoped_message_writer&& rhs) = delete; + + template<typename T> + std::ostream& operator<<(const T& val) + { + m_oss << val; + return m_oss; + } + + ~scoped_message_writer() + { + if (m_flush) + { + m_flush = false; + + LOG_PRINT(m_oss.str(), m_log_level) + + if (epee::log_space::console_color_default == m_color) + { + std::cout << m_oss.str(); + } + else + { + epee::log_space::set_console_color(m_color, m_bright); + std::cout << m_oss.str(); + epee::log_space::reset_console_color(); + } + std::cout << std::endl; + } + } +}; + +inline scoped_message_writer success_msg_writer() +{ + return scoped_message_writer(epee::log_space::console_color_green, false, std::string(), LOG_LEVEL_2); +} + +inline scoped_message_writer msg_writer(epee::log_space::console_colors color = epee::log_space::console_color_default) +{ + return scoped_message_writer(color, false, std::string(), LOG_LEVEL_2); +} + +inline scoped_message_writer fail_msg_writer() +{ + return scoped_message_writer(epee::log_space::console_color_red, true, "Error: ", LOG_LEVEL_0); +} + +} // namespace tools diff --git a/src/common/unordered_containers_boost_serialization.h b/src/common/unordered_containers_boost_serialization.h index f432a8cbd..563023624 100644 --- a/src/common/unordered_containers_boost_serialization.h +++ b/src/common/unordered_containers_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/util.cpp b/src/common/util.cpp index 0ad1f0791..7d39bc4f4 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -326,7 +326,7 @@ std::string get_nix_version_display_string() std::string config_folder; #ifdef WIN32 - config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CRYPTONOTE_NAME; + config_folder = get_special_folder_path(CSIDL_COMMON_APPDATA, true) + "\\" + CRYPTONOTE_NAME; #else std::string pathRet; char* pszHome = getenv("HOME"); diff --git a/src/common/util.h b/src/common/util.h index 51f0dad90..883fe1e0f 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -58,6 +58,18 @@ namespace tools */ std::string get_default_data_dir(); +#ifdef WIN32 + /** + * @brief + * + * @param nfolder + * @param iscreate + * + * @return + */ + std::string get_special_folder_path(int nfolder, bool iscreate); +#endif + /*! \brief Returns the OS version string * * \details This is a wrapper around the primitives diff --git a/src/common/varint.h b/src/common/varint.h index 5a73c2746..cd88e12a7 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/connectivity_tool/CMakeLists.txt b/src/connectivity_tool/CMakeLists.txt index b0178c70a..f36963d23 100644 --- a/src/connectivity_tool/CMakeLists.txt +++ b/src/connectivity_tool/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/connectivity_tool/conn_tool.cpp b/src/connectivity_tool/conn_tool.cpp index abddd5245..7506dba6f 100644 --- a/src/connectivity_tool/conn_tool.cpp +++ b/src/connectivity_tool/conn_tool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -49,6 +49,8 @@ namespace po = boost::program_options; using namespace cryptonote; using namespace nodetool; +unsigned int epee::g_test_dbg_lock_sleep = 0; + namespace { const command_line::arg_descriptor<std::string, true> arg_ip = {"ip", "set ip"}; diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 4afcab9c8..54dd99e26 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index b6c1bd2f5..63163b51b 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/blake256.h b/src/crypto/blake256.h index b6edbad9c..1545b2a9d 100644 --- a/src/crypto/blake256.h +++ b/src/crypto/blake256.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/chacha8.h b/src/crypto/chacha8.h index 0ce130834..9527e4016 100644 --- a/src/crypto/chacha8.h +++ b/src/crypto/chacha8.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 298399308..c26abacca 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 1a7bc8f45..e3b48fbb2 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index 8ee75d2fe..0bd1f9f9b 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index fa0199f20..01e35f399 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index f72a03f98..afbc9732e 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index a08f60dd4..a860a2076 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl.c b/src/crypto/groestl.c index 00bf987c9..e1c89cc3b 100644 --- a/src/crypto/groestl.c +++ b/src/crypto/groestl.c @@ -357,4 +357,4 @@ static int crypto_hash(unsigned char *out, return 0; } -*/
\ No newline at end of file +*/ diff --git a/src/crypto/groestl.h b/src/crypto/groestl.h index e2c6d2174..a9c907a19 100644 --- a/src/crypto/groestl.h +++ b/src/crypto/groestl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index 79e350bf3..d080a53db 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-blake.c b/src/crypto/hash-extra-blake.c index e9007b5d8..d8a7b7e22 100644 --- a/src/crypto/hash-extra-blake.c +++ b/src/crypto/hash-extra-blake.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-groestl.c b/src/crypto/hash-extra-groestl.c index 2b8a77965..6503a0624 100644 --- a/src/crypto/hash-extra-groestl.c +++ b/src/crypto/hash-extra-groestl.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-jh.c b/src/crypto/hash-extra-jh.c index d18ef3993..aaa80ca0f 100644 --- a/src/crypto/hash-extra-jh.c +++ b/src/crypto/hash-extra-jh.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-skein.c b/src/crypto/hash-extra-skein.c index 8818815af..6f18a300e 100644 --- a/src/crypto/hash-extra-skein.c +++ b/src/crypto/hash-extra-skein.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index ad5a3407a..5e2790a0c 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 9a628d0bd..d357ae93f 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 7b380c29d..2f91a5358 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index 61f75cecd..1d4b77be5 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.c b/src/crypto/random.c index 727659e82..6cffda2c0 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.h b/src/crypto/random.h index 85e55fd7d..0f5d8c9b1 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/skein_port.h b/src/crypto/skein_port.h index a2a8c45aa..6d25c0a93 100644 --- a/src/crypto/skein_port.h +++ b/src/crypto/skein_port.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 787545830..425737984 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -37,6 +37,10 @@ #include "hash-ops.h" #include "oaes_lib.h" +#if defined(__x86_64__) +// Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI +// Fall back to more portable code is down at the bottom + #include <emmintrin.h> #if defined(_MSC_VER) @@ -619,3 +623,163 @@ void cn_slow_hash(const void *data, size_t length, char *hash) hash_permutation(&state.hs); extra_hashes[state.hs.b[0] & 3](&state, 200, hash); } + +#else +// Portable implementation as a fallback + +void slow_hash_allocate_state(void) +{ + // Do nothing, this is just to maintain compatibility with the upgraded slow-hash.c + return; +} + +void slow_hash_free_state(void) +{ + // As above + return; +} + +static void (*const extra_hashes[4])(const void *, size_t, char *) = { + hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein +}; + +#define MEMORY (1 << 21) /* 2 MiB */ +#define ITER (1 << 20) +#define AES_BLOCK_SIZE 16 +#define AES_KEY_SIZE 32 /*16*/ +#define INIT_SIZE_BLK 8 +#define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE) + +extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); +extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); + +static size_t e2i(const uint8_t* a, size_t count) { return (*((uint64_t*)a) / AES_BLOCK_SIZE) & (count - 1); } + +static void mul(const uint8_t* a, const uint8_t* b, uint8_t* res) { + uint64_t a0, b0; + uint64_t hi, lo; + + a0 = SWAP64LE(((uint64_t*)a)[0]); + b0 = SWAP64LE(((uint64_t*)b)[0]); + lo = mul128(a0, b0, &hi); + ((uint64_t*)res)[0] = SWAP64LE(hi); + ((uint64_t*)res)[1] = SWAP64LE(lo); +} + +static void sum_half_blocks(uint8_t* a, const uint8_t* b) { + uint64_t a0, a1, b0, b1; + + a0 = SWAP64LE(((uint64_t*)a)[0]); + a1 = SWAP64LE(((uint64_t*)a)[1]); + b0 = SWAP64LE(((uint64_t*)b)[0]); + b1 = SWAP64LE(((uint64_t*)b)[1]); + a0 += b0; + a1 += b1; + ((uint64_t*)a)[0] = SWAP64LE(a0); + ((uint64_t*)a)[1] = SWAP64LE(a1); +} +#define U64(x) ((uint64_t *) (x)) + +static void copy_block(uint8_t* dst, const uint8_t* src) { + memcpy(dst, src, AES_BLOCK_SIZE); +} + +static void swap_blocks(uint8_t *a, uint8_t *b){ + uint64_t t[2]; + U64(t)[0] = U64(a)[0]; + U64(t)[1] = U64(a)[1]; + U64(a)[0] = U64(b)[0]; + U64(a)[1] = U64(b)[1]; + U64(b)[0] = U64(t)[0]; + U64(b)[1] = U64(t)[1]; +} + +static void xor_blocks(uint8_t* a, const uint8_t* b) { + size_t i; + for (i = 0; i < AES_BLOCK_SIZE; i++) { + a[i] ^= b[i]; + } +} + +#pragma pack(push, 1) +union cn_slow_hash_state { + union hash_state hs; + struct { + uint8_t k[64]; + uint8_t init[INIT_SIZE_BYTE]; + }; +}; +#pragma pack(pop) + +void cn_slow_hash(const void *data, size_t length, char *hash) { + uint8_t long_state[MEMORY]; + union cn_slow_hash_state state; + uint8_t text[INIT_SIZE_BYTE]; + uint8_t a[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE]; + uint8_t c[AES_BLOCK_SIZE]; + uint8_t d[AES_BLOCK_SIZE]; + size_t i, j; + uint8_t aes_key[AES_KEY_SIZE]; + oaes_ctx *aes_ctx; + + hash_process(&state.hs, data, length); + memcpy(text, state.init, INIT_SIZE_BYTE); + memcpy(aes_key, state.hs.b, AES_KEY_SIZE); + aes_ctx = (oaes_ctx *) oaes_alloc(); + + oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); + for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { + for (j = 0; j < INIT_SIZE_BLK; j++) { + aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); + } + memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + } + + for (i = 0; i < 16; i++) { + a[i] = state.k[ i] ^ state.k[32 + i]; + b[i] = state.k[16 + i] ^ state.k[48 + i]; + } + + for (i = 0; i < ITER / 2; i++) { + /* Dependency chain: address -> read value ------+ + * written value <-+ hard function (AES or MUL) <+ + * next address <-+ + */ + /* Iteration 1 */ + j = e2i(a, MEMORY / AES_BLOCK_SIZE); + copy_block(c, &long_state[j * AES_BLOCK_SIZE]); + aesb_single_round(c, c, a); + xor_blocks(b, c); + swap_blocks(b, c); + copy_block(&long_state[j * AES_BLOCK_SIZE], c); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); + swap_blocks(a, b); + /* Iteration 2 */ + j = e2i(a, MEMORY / AES_BLOCK_SIZE); + copy_block(c, &long_state[j * AES_BLOCK_SIZE]); + mul(a, c, d); + sum_half_blocks(b, d); + swap_blocks(b, c); + xor_blocks(b, c); + copy_block(&long_state[j * AES_BLOCK_SIZE], c); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); + swap_blocks(a, b); + } + + memcpy(text, state.init, INIT_SIZE_BYTE); + oaes_key_import_data(aes_ctx, &state.hs.b[32], AES_KEY_SIZE); + for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { + for (j = 0; j < INIT_SIZE_BLK; j++) { + xor_blocks(&text[j * AES_BLOCK_SIZE], &long_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); + aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); + } + } + memcpy(state.init, text, INIT_SIZE_BYTE); + hash_permutation(&state.hs); + /*memcpy(hash, &state, 32);*/ + extra_hashes[state.hs.b[0] & 3](&state, 200, hash); + oaes_free((OAES_CTX **) &aes_ctx); +} + +#endif diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 5a84c8688..a75f91b80 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 7864b974e..e5df844e7 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -33,6 +33,8 @@ #include <string> #include <boost/uuid/uuid.hpp> +#define CRYPTONOTE_DNS_TIMEOUT_MS 20000 + #define CRYPTONOTE_MAX_BLOCK_NUMBER 500000000 #define CRYPTONOTE_MAX_BLOCK_SIZE 500000000 // block header blob limit, never used! #define CRYPTONOTE_GETBLOCKTEMPLATE_MAX_BLOCK_SIZE 196608 //size of block (bytes) that is the maximum that miners will produce @@ -49,6 +51,7 @@ // MONEY_SUPPLY - total number coins to be generated #define MONEY_SUPPLY ((uint64_t)(-1)) #define EMISSION_SPEED_FACTOR (20) +#define FINAL_SUBSIDY_PER_MINUTE ((uint64_t)300000000000) // 3 * pow(10, 11) #define CRYPTONOTE_REWARD_BLOCKS_WINDOW 100 #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE 20000 //size of block (bytes) after which reward for block calculated using block size @@ -59,9 +62,6 @@ #define FEE_PER_KB ((uint64_t)10000000000) // pow(10, 10) -// temporarily to allow backward compatibility during the switch to per-kb -//#define MINING_ALLOWED_LEGACY_FEE ((uint64_t)100000000000) // pow(10, 11) - #define ORPHANED_BLOCKS_MAX_COUNT 100 diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 3c2e097c1..e144a93a2 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -29,6 +29,7 @@ set(cryptonote_core_sources account.cpp blockchain_storage.cpp + blockchain.cpp checkpoints.cpp checkpoints_create.cpp cryptonote_basic_impl.cpp @@ -45,6 +46,7 @@ set(cryptonote_core_private_headers account_boost_serialization.h blockchain_storage.h blockchain_storage_boost_serialization.h + blockchain.h checkpoints.h checkpoints_create.h connection_context.h @@ -70,6 +72,8 @@ target_link_libraries(cryptonote_core LINK_PUBLIC common crypto + otshell_utils + blockchain_db ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} diff --git a/src/cryptonote_core/account.cpp b/src/cryptonote_core/account.cpp index 9f6c91026..a4de8fb25 100644 --- a/src/cryptonote_core/account.cpp +++ b/src/cryptonote_core/account.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/account.h b/src/cryptonote_core/account.h index dd6618542..46f14fcbb 100644 --- a/src/cryptonote_core/account.h +++ b/src/cryptonote_core/account.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/account_boost_serialization.h b/src/cryptonote_core/account_boost_serialization.h index f9ecea895..cddc05e36 100644 --- a/src/cryptonote_core/account_boost_serialization.h +++ b/src/cryptonote_core/account_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp new file mode 100644 index 000000000..0261a5614 --- /dev/null +++ b/src/cryptonote_core/blockchain.cpp @@ -0,0 +1,2342 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include <algorithm> +#include <cstdio> +#include <boost/archive/binary_oarchive.hpp> +#include <boost/archive/binary_iarchive.hpp> +#include <boost/filesystem.hpp> + +#include "include_base_utils.h" +#include "cryptonote_basic_impl.h" +#include "tx_pool.h" +#include "blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_format_utils.h" +#include "cryptonote_boost_serialization.h" +#include "cryptonote_config.h" +#include "miner.h" +#include "misc_language.h" +#include "profile_tools.h" +#include "file_io_utils.h" +#include "common/boost_serialization_helper.h" +#include "warnings.h" +#include "crypto/hash.h" +#include "cryptonote_core/checkpoints_create.h" +//#include "serialization/json_archive.h" + +/* TODO: + * Clean up code: + * Possibly change how outputs are referred to/indexed in blockchain and wallets + * + */ + +using namespace cryptonote; +using epee::string_tools::pod_to_hex; + +DISABLE_VS_WARNINGS(4267) + +//------------------------------------------------------------------ +Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) +{ + LOG_PRINT_L3("Blockchain::" << __func__); +} +//------------------------------------------------------------------ +//TODO: is this still needed? I don't think so - tewinget +template<class archive_t> +void Blockchain::serialize(archive_t & ar, const unsigned int version) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + if(version < 11) + return; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + ar & m_blocks; + ar & m_blocks_index; + ar & m_transactions; + ar & m_spent_keys; + ar & m_alternative_chains; + ar & m_outputs; + ar & m_invalid_blocks; + ar & m_current_block_cumul_sz_limit; + /*serialization bug workaround*/ + if(version > 11) + { + uint64_t total_check_count = m_db->height() + m_blocks_index.size() + m_transactions.size() + m_spent_keys.size() + m_alternative_chains.size() + m_outputs.size() + m_invalid_blocks.size() + m_current_block_cumul_sz_limit; + if(archive_t::is_saving::value) + { + ar & total_check_count; + }else + { + uint64_t total_check_count_loaded = 0; + ar & total_check_count_loaded; + if(total_check_count != total_check_count_loaded) + { + LOG_ERROR("Blockchain storage data corruption detected. total_count loaded from file = " << total_check_count_loaded << ", expected = " << total_check_count); + + LOG_PRINT_L0("Blockchain storage:" << std::endl << + "m_blocks: " << m_db->height() << std::endl << + "m_blocks_index: " << m_blocks_index.size() << std::endl << + "m_transactions: " << m_transactions.size() << std::endl << + "m_spent_keys: " << m_spent_keys.size() << std::endl << + "m_alternative_chains: " << m_alternative_chains.size() << std::endl << + "m_outputs: " << m_outputs.size() << std::endl << + "m_invalid_blocks: " << m_invalid_blocks.size() << std::endl << + "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); + + throw std::runtime_error("Blockchain data corruption"); + } + } + } + + + LOG_PRINT_L3("Blockchain storage:" << std::endl << + "m_blocks: " << m_db->height() << std::endl << + "m_blocks_index: " << m_blocks_index.size() << std::endl << + "m_transactions: " << m_transactions.size() << std::endl << + "m_spent_keys: " << m_spent_keys.size() << std::endl << + "m_alternative_chains: " << m_alternative_chains.size() << std::endl << + "m_outputs: " << m_outputs.size() << std::endl << + "m_invalid_blocks: " << m_invalid_blocks.size() << std::endl << + "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); +} +//------------------------------------------------------------------ +bool Blockchain::have_tx(const crypto::hash &id) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->tx_exists(id); +} +//------------------------------------------------------------------ +bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->has_key_image(key_im); +} +//------------------------------------------------------------------ +// This function makes sure that each "input" in an input (mixins) exists +// and collects the public key for each from the transaction it was included in +// via the visitor passed to it. +template<class visitor_t> +bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // verify that the input has key offsets (that it exists properly, really) + if(!tx_in_to_key.key_offsets.size()) + return false; + + // cryptonote_format_utils uses relative offsets for indexing to the global + // outputs list. that is to say that absolute offset #2 is absolute offset + // #1 plus relative offset #2. + // TODO: Investigate if this is necessary / why this is done. + std::vector<uint64_t> absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); + + + //std::vector<std::pair<crypto::hash, size_t> >& amount_outs_vec = it->second; + size_t count = 0; + for (const uint64_t& i : absolute_offsets) + { + try + { + // get tx hash and output index for output + auto output_index = m_db->get_output_tx_and_index(tx_in_to_key.amount, i); + + // get tx that output is from + auto tx = m_db->get_tx(output_index.first); + + // make sure output index is within range for the given transaction + if (output_index.second >= tx.vout.size()) + { + LOG_PRINT_L0("Output does not exist. tx = " << output_index.first << ", index = " << output_index.second); + return false; + } + + // call to the passed boost visitor to grab the public key for the output + if(!vis.handle_output(tx, tx.vout[output_index.second])) + { + LOG_PRINT_L0("Failed to handle_output for output no = " << count << ", with absolute offset " << i); + return false; + } + + // if on last output and pmax_related_block_height not null pointer + if(++count == absolute_offsets.size() && pmax_related_block_height) + { + // set *pmax_related_block_height to tx block height for this output + auto h = m_db->get_tx_block_height(output_index.first); + if(*pmax_related_block_height < h) + { + *pmax_related_block_height = h; + } + } + + } + catch (const OUTPUT_DNE& e) + { + LOG_PRINT_L0("Output does not exist: " << e.what()); + return false; + } + catch (const TX_DNE& e) + { + LOG_PRINT_L0("Transaction does not exist: " << e.what()); + return false; + } + + } + + return true; +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_current_blockchain_height() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->height(); +} +//------------------------------------------------------------------ +//FIXME: possibly move this into the constructor, to avoid accidentally +// dereferencing a null BlockchainDB pointer +bool Blockchain::init(BlockchainDB* db, const bool testnet) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + if (db == nullptr) + { + LOG_ERROR("Attempted to init Blockchain with null DB"); + return false; + } + if (!db->is_open()) + { + LOG_ERROR("Attempted to init Blockchain with unopened DB"); + return false; + } + + m_db = db; + + // if the blockchain is new, add the genesis block + // this feels kinda kludgy to do it this way, but can be looked at later. + // TODO: add function to create and store genesis block, + // taking testnet into account + if(!m_db->height()) + { + LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); + block bl = boost::value_initialized<block>(); + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + if (testnet) + { + generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); + } + else + { + generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); + } + add_new_block(bl, bvc); + CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); + } + // TODO: if blockchain load successful, verify blockchain against both + // hard-coded and runtime-loaded (and enforced) checkpoints. + else + { + } + + // check how far behind we are + uint64_t top_block_timestamp = m_db->get_top_block_timestamp(); + uint64_t timestamp_diff = time(NULL) - top_block_timestamp; + + // genesis block has no timestamp, could probably change it to have timestamp of 1341378000... + if(!top_block_timestamp) + timestamp_diff = time(NULL) - 1341378000; + LOG_PRINT_GREEN("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block(), LOG_LEVEL_0); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::store_blockchain() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + // TODO: make sure if this throws that it is not simply ignored higher + // up the call stack + try + { + m_db->sync(); + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Error syncing blockchain db: ") + e.what() + "-- shutting down now to prevent issues!"); + throw; + } + catch (...) + { + LOG_PRINT_L0("There was an issue storing the blockchain, shutting down now to prevent issues!"); + throw; + } + LOG_PRINT_L0("Blockchain stored OK."); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::deinit() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + // as this should be called if handling a SIGSEGV, need to check + // if m_db is a NULL pointer (and thus may have caused the illegal + // memory operation), otherwise we may cause a loop. + if (m_db == NULL) + { + throw new DB_ERROR("The db pointer is null in Blockchain, the blockchain may be corrupt!"); + } + + try + { + m_db->close(); + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Error closing blockchain db: ") + e.what()); + } + catch (...) + { + LOG_PRINT_L0("There was an issue closing/storing the blockchain, shutting down now to prevent issues!"); + } + + delete m_db; + return true; +} +//------------------------------------------------------------------ +// This function tells BlockchainDB to remove the top block from the +// blockchain and then returns all transactions (except the miner tx, of course) +// from it to the tx_pool +block Blockchain::pop_block_from_blockchain() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + block popped_block; + std::vector<transaction> popped_txs; + + try + { + m_db->pop_block(popped_block, popped_txs); + } + // anything that could cause this to throw is likely catastrophic, + // so we re-throw + catch (const std::exception& e) + { + LOG_ERROR("Error popping block from blockchain: " << e.what()); + throw; + } + catch (...) + { + LOG_ERROR("Error popping block from blockchain, throwing!"); + throw; + } + + // return transactions from popped block to the tx_pool + for (transaction& tx : popped_txs) + { + if (!is_coinbase(tx)) + { + cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool r = m_tx_pool.add_tx(tx, tvc, true); + if (!r) + { + LOG_ERROR("Error returning transaction to tx_pool"); + } + } + } + + return popped_block; +} +//------------------------------------------------------------------ +bool Blockchain::reset_and_set_genesis_block(const block& b) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_transactions.clear(); + m_spent_keys.clear(); + m_blocks.clear(); + m_blocks_index.clear(); + m_alternative_chains.clear(); + m_outputs.clear(); + m_db->reset(); + + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + add_new_block(b, bvc); + return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; +} +//------------------------------------------------------------------ +//TODO: move to BlockchainDB subclass +bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + struct purge_transaction_visitor: public boost::static_visitor<bool> + { + key_images_container& m_spent_keys; + bool m_strict_check; + purge_transaction_visitor(key_images_container& spent_keys, bool strict_check):m_spent_keys(spent_keys), m_strict_check(strict_check){} + + bool operator()(const txin_to_key& inp) const + { + //const crypto::key_image& ki = inp.k_image; + auto r = m_spent_keys.find(inp.k_image); + if(r != m_spent_keys.end()) + { + m_spent_keys.erase(r); + }else + { + CHECK_AND_ASSERT_MES(!m_strict_check, false, "purge_block_data_from_blockchain: key image in transaction not found"); + } + return true; + } + bool operator()(const txin_gen& inp) const + { + return true; + } + bool operator()(const txin_to_script& tx) const + { + return false; + } + + bool operator()(const txin_to_scripthash& tx) const + { + return false; + } + }; + + BOOST_FOREACH(const txin_v& in, tx.vin) + { + bool r = boost::apply_visitor(purge_transaction_visitor(m_spent_keys, strict_check), in); + CHECK_AND_ASSERT_MES(!strict_check || r, false, "failed to process purge_transaction_visitor"); + } + return true; +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_tail_id(uint64_t& height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + height = m_db->height() - 1; + return get_tail_id(); +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_tail_id() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->top_block_hash(); +} +//------------------------------------------------------------------ +/*TODO: this function was...poorly written. As such, I'm not entirely + * certain on what it was supposed to be doing. Need to look into this, + * but it doesn't seem terribly important just yet. + * + * puts into list <ids> a list of hashes representing certain blocks + * from the blockchain in reverse chronological order + * + * the blocks chosen, at the time of this writing, are: + * the most recent 11 + * powers of 2 less recent from there, so 13, 17, 25, etc... + * + */ +bool Blockchain::get_short_chain_history(std::list<crypto::hash>& ids) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + uint64_t i = 0; + uint64_t current_multiplier = 1; + uint64_t sz = m_db->height(); + + if(!sz) + return true; + + bool genesis_included = false; + uint64_t current_back_offset = 1; + while(current_back_offset < sz) + { + ids.push_back(m_db->get_block_hash_from_height(sz - current_back_offset)); + + if(sz-current_back_offset == 0) + { + genesis_included = true; + } + if(i < 10) + { + ++current_back_offset; + } + else + { + current_multiplier *= 2; + current_back_offset += current_multiplier; + } + ++i; + } + + if (!genesis_included) + { + ids.push_back(m_db->get_block_hash_from_height(0)); + } + + return true; +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_block_id_by_height(uint64_t height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + try + { + return m_db->get_block_hash_from_height(height); + } + catch (const BLOCK_DNE& e) + { + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height: ") + e.what()); + throw; + } + catch (...) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height")); + throw; + } + return null_hash; +} +//------------------------------------------------------------------ +bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // try to find block in main chain + try + { + blk = m_db->get_block(h); + return true; + } + // try to find block in alternative chain + catch (const BLOCK_DNE& e) + { + blocks_ext_by_hash::const_iterator it_alt = m_alternative_chains.find(h); + if (m_alternative_chains.end() != it_alt) { + blk = it_alt->second.bl; + return true; + } + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block by hash: ") + e.what()); + throw; + } + catch (...) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by hash")); + throw; + } + + return false; +} +//------------------------------------------------------------------ +//FIXME: this function does not seem to be called from anywhere, but +// if it ever is, should probably change std::list for std::vector +void Blockchain::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (auto& a : m_db->get_hashes_range(0, m_db->height() - 1)) + { + main.push_back(a); + } + + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_alternative_chains) + alt.push_back(v.first); + + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_invalid_blocks) + invalid.push_back(v.first); +} +//------------------------------------------------------------------ +// This function aggregates the cumulative difficulties and timestamps of the +// last DIFFICULTY_BLOCKS_COUNT blocks and passes them to next_difficulty, +// returning the result of that call. Ignores the genesis block, and can use +// less blocks than desired if there aren't enough. +difficulty_type Blockchain::get_difficulty_for_next_block() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + std::vector<uint64_t> timestamps; + std::vector<difficulty_type> cumulative_difficulties; + auto h = m_db->height(); + + size_t offset = h - std::min<size_t>(h, static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT)); + + if (offset == 0) + { + ++offset; + } + + for(; offset < h; offset++) + { + timestamps.push_back(m_db->get_block_timestamp(offset)); + cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(offset)); + } + return next_difficulty(timestamps, cumulative_difficulties); +} +//------------------------------------------------------------------ +// This function removes blocks from the blockchain until it gets to the +// position where the blockchain switch started and then re-adds the blocks +// that had been removed. +bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // remove blocks from blockchain until we get back to where we should be. + while (m_db->height() != rollback_height) + { + pop_block_from_blockchain(); + } + + //return back original chain + for (auto& bl : original_chain) + { + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + bool r = handle_block_to_main_chain(bl, bvc); + CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC! failed to add (again) block while chain switching during the rollback!"); + } + + LOG_PRINT_L1("Rollback to height " << rollback_height << " was successful."); + if (original_chain.size()) + { + LOG_PRINT_L1("Restoration to previous blockchain successful as well."); + } + return true; +} +//------------------------------------------------------------------ +// This function attempts to switch to an alternate chain, returning +// boolean based on success therein. +bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if empty alt chain passed (not sure how that could happen), return false + CHECK_AND_ASSERT_MES(alt_chain.size(), false, "switch_to_alternative_blockchain: empty chain passed"); + + // verify that main chain has front of alt chain's parent block + if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + { + LOG_ERROR("Attempting to move to an alternate chain, but it doesn't appear to connect to the main chain!"); + return false; + } + + // pop blocks from the blockchain until the top block is the parent + // of the front block of the alt chain. + std::list<block> disconnected_chain; + while (m_db->top_block_hash() != alt_chain.front()->second.bl.prev_id) + { + block b = pop_block_from_blockchain(); + disconnected_chain.push_front(b); + } + + auto split_height = m_db->height(); + + //connecting new alternative chain + for(auto alt_ch_iter = alt_chain.begin(); alt_ch_iter != alt_chain.end(); alt_ch_iter++) + { + auto ch_ent = *alt_ch_iter; + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + + // add block to main chain + bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc); + + // if adding block to main chain failed, rollback to previous state and + // return false + if(!r || !bvc.m_added_to_main_chain) + { + LOG_PRINT_L1("Failed to switch to alternative blockchain"); + + // rollback_blockchain_switching should be moved to two different + // functions: rollback and apply_chain, but for now we pretend it is + // just the latter (because the rollback was done above). + rollback_blockchain_switching(disconnected_chain, m_db->height()); + + // FIXME: Why do we keep invalid blocks around? Possibly in case we hear + // about them again so we can immediately dismiss them, but needs some + // looking into. + add_block_as_invalid(ch_ent->second, get_block_hash(ch_ent->second.bl)); + LOG_PRINT_L1("The block was inserted as invalid while connecting new alternative chain, block_id: " << get_block_hash(ch_ent->second.bl)); + m_alternative_chains.erase(ch_ent); + + for(auto alt_ch_to_orph_iter = ++alt_ch_iter; alt_ch_to_orph_iter != alt_chain.end(); alt_ch_to_orph_iter++) + { + add_block_as_invalid((*alt_ch_iter)->second, (*alt_ch_iter)->first); + m_alternative_chains.erase(*alt_ch_to_orph_iter); + } + return false; + } + } + + // if we're to keep the disconnected blocks, add them as alternates + if(!discard_disconnected_chain) + { + //pushing old chain as alternative chain + for (auto& old_ch_ent : disconnected_chain) + { + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc); + if(!r) + { + LOG_PRINT_L1("Failed to push ex-main chain blocks to alternative chain "); + // previously this would fail the blockchain switching, but I don't + // think this is bad enough to warrant that. + } + } + } + + //removing alt_chain entries from alternative chain + BOOST_FOREACH(auto ch_ent, alt_chain) + { + m_alternative_chains.erase(ch_ent); + } + + LOG_PRINT_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height(), LOG_LEVEL_0); + return true; +} +//------------------------------------------------------------------ +// This function calculates the difficulty target for the block being added to +// an alternate chain. +difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::vector<uint64_t> timestamps; + std::vector<difficulty_type> cumulative_difficulties; + + // if the alt chain isn't long enough to calculate the difficulty target + // based on its blocks alone, need to get more blocks from the main chain + if(alt_chain.size()< DIFFICULTY_BLOCKS_COUNT) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // Figure out start and stop offsets for main chain blocks + size_t main_chain_stop_offset = alt_chain.size() ? alt_chain.front()->second.height : bei.height; + size_t main_chain_count = DIFFICULTY_BLOCKS_COUNT - std::min(static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT), alt_chain.size()); + main_chain_count = std::min(main_chain_count, main_chain_stop_offset); + size_t main_chain_start_offset = main_chain_stop_offset - main_chain_count; + + if(!main_chain_start_offset) + ++main_chain_start_offset; //skip genesis block + + // get difficulties and timestamps from relevant main chain blocks + for(; main_chain_start_offset < main_chain_stop_offset; ++main_chain_start_offset) + { + timestamps.push_back(m_db->get_block_timestamp(main_chain_start_offset)); + cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(main_chain_start_offset)); + } + + // make sure we haven't accidentally grabbed too many blocks...maybe don't need this check? + CHECK_AND_ASSERT_MES((alt_chain.size() + timestamps.size()) <= DIFFICULTY_BLOCKS_COUNT, false, + "Internal error, alt_chain.size()[" << alt_chain.size() + << "] + vtimestampsec.size()[" << timestamps.size() + << "] NOT <= DIFFICULTY_WINDOW[]" << DIFFICULTY_BLOCKS_COUNT + ); + + for (auto it : alt_chain) + { + timestamps.push_back(it->second.bl.timestamp); + cumulative_difficulties.push_back(it->second.cumulative_difficulty); + } + } + // if the alt chain is long enough for the difficulty calc, grab difficulties + // and timestamps from it alone + else + { + timestamps.resize(static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT)); + cumulative_difficulties.resize(static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT)); + size_t count = 0; + size_t max_i = timestamps.size()-1; + // get difficulties and timestamps from most recent blocks in alt chain + BOOST_REVERSE_FOREACH(auto it, alt_chain) + { + timestamps[max_i - count] = it->second.bl.timestamp; + cumulative_difficulties[max_i - count] = it->second.cumulative_difficulty; + count++; + if(count >= DIFFICULTY_BLOCKS_COUNT) + break; + } + } + + // calculate the difficulty target for the block and return it + return next_difficulty(timestamps, cumulative_difficulties); +} +//------------------------------------------------------------------ +// This function does a sanity check on basic things that all miner +// transactions have in common, such as: +// one input, of type txin_gen, with height set to the block's height +// correct miner tx unlock time +// a non-overflowing tx amount (dubious necessity on this check) +bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); + CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); + if(boost::get<txin_gen>(b.miner_tx.vin[0]).height != height) + { + LOG_PRINT_RED_L1("The miner transaction in block has invalid height: " << boost::get<txin_gen>(b.miner_tx.vin[0]).height << ", expected: " << height); + return false; + } + CHECK_AND_ASSERT_MES(b.miner_tx.unlock_time == height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, + false, + "coinbase transaction transaction has the wrong unlock time=" << b.miner_tx.unlock_time << ", expected " << height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW); + + + //check outs overflow + //NOTE: not entirely sure this is necessary, given that this function is + // designed simply to make sure the total amount for a transaction + // does not overflow a uint64_t, and this transaction *is* a uint64_t... + if(!check_outs_overflow(b.miner_tx)) + { + LOG_PRINT_RED_L1("miner transaction has money overflow in block " << get_block_hash(b)); + return false; + } + + return true; +} +//------------------------------------------------------------------ +// This function validates the miner transaction reward +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + //validate reward + uint64_t money_in_use = 0; + BOOST_FOREACH(auto& o, b.miner_tx.vout) + money_in_use += o.amount; + + std::vector<size_t> last_blocks_sizes; + get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward)) { + LOG_PRINT_L1("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + return false; + } + if(base_reward + fee < money_in_use) + { + LOG_PRINT_L1("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + return false; + } + if(base_reward + fee != money_in_use) + { + LOG_PRINT_L1("coinbase transaction doesn't use full amount of block reward: spent: " + << money_in_use << ", block reward " << base_reward + fee << "(" << base_reward << "+" << fee << ")"); + return false; + } + return true; +} +//------------------------------------------------------------------ +// get the block sizes of the last <count> blocks, starting at <from_height> +// and return by reference <sz>. +void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto h = m_db->height(); + + // this function is meaningless for an empty blockchain...granted it should never be empty + if(h == 0) + return; + + // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) + size_t start_offset = h - std::min<size_t>(h, count); + for(size_t i = start_offset; i < h; i++) + { + sz.push_back(m_db->get_block_size(i)); + } +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_current_cumulative_blocksize_limit() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + return m_current_block_cumul_sz_limit; +} +//------------------------------------------------------------------ +//TODO: This function only needed minor modification to work with BlockchainDB, +// and *works*. As such, to reduce the number of things that might break +// in moving to BlockchainDB, this function will remain otherwise +// unchanged for the time being. +// +// This function makes a new block for a miner to mine the hash for +// +// FIXME: this codebase references #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) +// in a lot of places. That flag is not referenced in any of the code +// nor any of the makefiles, howeve. Need to look into whether or not it's +// necessary at all. +bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + size_t median_size; + uint64_t already_generated_coins; + + CRITICAL_REGION_BEGIN(m_blockchain_lock); + b.major_version = CURRENT_BLOCK_MAJOR_VERSION; + b.minor_version = CURRENT_BLOCK_MINOR_VERSION; + b.prev_id = get_tail_id(); + b.timestamp = time(NULL); + + height = m_db->height(); + diffic = get_difficulty_for_next_block(); + CHECK_AND_ASSERT_MES(diffic, false, "difficulty owverhead."); + + median_size = m_current_block_cumul_sz_limit / 2; + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + + CRITICAL_REGION_END(); + + size_t txs_size; + uint64_t fee; + if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { + return false; + } +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + size_t real_txs_size = 0; + uint64_t real_fee = 0; + CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); + BOOST_FOREACH(crypto::hash &cur_hash, b.tx_hashes) { + auto cur_res = m_tx_pool.m_transactions.find(cur_hash); + if (cur_res == m_tx_pool.m_transactions.end()) { + LOG_ERROR("Creating block template: error: transaction not found"); + continue; + } + tx_memory_pool::tx_details &cur_tx = cur_res->second; + real_txs_size += cur_tx.blob_size; + real_fee += cur_tx.fee; + if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) { + LOG_ERROR("Creating block template: error: invalid transaction size"); + } + uint64_t inputs_amount; + if (!get_inputs_money_amount(cur_tx.tx, inputs_amount)) { + LOG_ERROR("Creating block template: error: cannot get inputs amount"); + } else if (cur_tx.fee != inputs_amount - get_outs_money_amount(cur_tx.tx)) { + LOG_ERROR("Creating block template: error: invalid fee"); + } + } + if (txs_size != real_txs_size) { + LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + } + if (fee != real_fee) { + LOG_ERROR("Creating block template: error: wrongly calculated fee"); + } + CRITICAL_REGION_END(); + LOG_PRINT_L1("Creating block template: height " << height << + ", median size " << median_size << + ", already generated coins " << already_generated_coins << + ", transaction size " << txs_size << + ", fee " << fee); +#endif + + /* + two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know + block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + */ + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size + bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, first chance"); + size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << + ", cumulative size " << cumulative_size); +#endif + for (size_t try_count = 0; try_count != 10; ++try_count) { + r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + + CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, second chance"); + size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); + if (coinbase_blob_size > cumulative_size - txs_size) { + cumulative_size = txs_size + coinbase_blob_size; +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << cumulative_size << " is greater then before"); +#endif + continue; + } + + if (coinbase_blob_size < cumulative_size - txs_size) { + size_t delta = cumulative_size - txs_size - coinbase_blob_size; +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << txs_size + coinbase_blob_size << + " is less then before, adding " << delta << " zero bytes"); +#endif + b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); + //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. + if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { + CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); + if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + LOG_PRINT_RED("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1 , LOG_LEVEL_2); + cumulative_size += delta - 1; + continue; + } + LOG_PRINT_GREEN("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count, LOG_LEVEL_1); + } + } + CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << cumulative_size << " is now good"); +#endif + return true; + } + LOG_ERROR("Failed to create_block_template with " << 10 << " tries"); + return false; +} +//------------------------------------------------------------------ +// for an alternate chain, get the timestamps from the main chain to complete +// the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. +bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + + if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) + return true; + + CRITICAL_REGION_LOCAL(m_blockchain_lock); + size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); + CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); + size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + while (start_top_height != stop_offset) + { + timestamps.push_back(m_db->get_block_timestamp(start_top_height)); + --start_top_height; + } + return true; +} +//------------------------------------------------------------------ +// If a block is to be added and its parent block is not the current +// main chain top block, then we need to see if we know about its parent block. +// If its parent block is part of a known forked chain, then we need to see +// if that chain is long enough to become the main chain and re-org accordingly +// if so. If not, we need to hang on to the block in case it becomes part of +// a long forked chain eventually. +bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + uint64_t block_height = get_block_height(b); + if(0 == block_height) + { + LOG_PRINT_L1("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative), but miner tx says height is 0."); + bvc.m_verifivation_failed = true; + return false; + } + // this basically says if the blockchain is smaller than the first + // checkpoint then alternate blocks are allowed. Alternatively, if the + // last checkpoint *before* the end of the current chain is also before + // the block to be added, then this is fine. + if (!m_checkpoints.is_alternative_block_allowed(get_current_blockchain_height(), block_height)) + { + LOG_PRINT_RED_L1("Block with id: " << id + << std::endl << " can't be accepted for alternative chain, block height: " << block_height + << std::endl << " blockchain height: " << get_current_blockchain_height()); + bvc.m_verifivation_failed = true; + return false; + } + + //block is not related with head of main chain + //first of all - look in alternative chains container + auto it_prev = m_alternative_chains.find(b.prev_id); + bool parent_in_main = m_db->block_exists(b.prev_id); + if(it_prev != m_alternative_chains.end() || parent_in_main) + { + //we have new block in alternative chain + + //build alternative subchain, front -> mainchain, back -> alternative head + blocks_ext_by_hash::iterator alt_it = it_prev; //m_alternative_chains.find() + std::list<blocks_ext_by_hash::iterator> alt_chain; + std::vector<uint64_t> timestamps; + while(alt_it != m_alternative_chains.end()) + { + alt_chain.push_front(alt_it); + timestamps.push_back(alt_it->second.bl.timestamp); + alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id); + } + + // if block to be added connects to known blocks that aren't part of the + // main chain -- that is, if we're adding on to an alternate chain + if(alt_chain.size()) + { + // make sure alt chain doesn't somehow start past the end of the main chain + CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); + + // make sure that the blockchain contains the block that should connect + // this alternate chain with it. + if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + { + LOG_PRINT_L1("alternate chain does not appear to connect to main chain..."); + return false; + } + + // make sure block connects correctly to the main chain + auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1); + CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain"); + complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps); + } + // if block not associated with known alternate chain + else + { + // if block parent is not part of main chain or an alternate chain, + // we ignore it + CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition it_main_prev != m_blocks_index.end()"); + + complete_timestamps_vector(m_db->get_block_height(b.prev_id), timestamps); + } + + // verify that the block's timestamp is within the acceptable range + // (not earlier than the median of the last X blocks) + if(!check_block_timestamp(timestamps, b)) + { + LOG_PRINT_RED_L1("Block with id: " << id + << std::endl << " for alternative chain, has invalid timestamp: " << b.timestamp); + bvc.m_verifivation_failed = true; + return false; + } + + // FIXME: consider moving away from block_extended_info at some point + block_extended_info bei = boost::value_initialized<block_extended_info>(); + bei.bl = b; + bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(b.prev_id) + 1; + + bool is_a_checkpoint; + if(!m_checkpoints.check_block(bei.height, id, is_a_checkpoint)) + { + LOG_ERROR("CHECKPOINT VALIDATION FAILED"); + bvc.m_verifivation_failed = true; + return false; + } + + // Check the block's hash against the difficulty target for its alt chain + m_is_in_checkpoint_zone = false; + difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei); + CHECK_AND_ASSERT_MES(current_diff, false, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!"); + crypto::hash proof_of_work = null_hash; + get_block_longhash(bei.bl, proof_of_work, bei.height); + if(!check_hash(proof_of_work, current_diff)) + { + LOG_PRINT_RED_L1("Block with id: " << id + << std::endl << " for alternative chain, does not have enough proof of work: " << proof_of_work + << std::endl << " expected difficulty: " << current_diff); + bvc.m_verifivation_failed = true; + return false; + } + + if(!prevalidate_miner_transaction(b, bei.height)) + { + LOG_PRINT_RED_L1("Block with id: " << epee::string_tools::pod_to_hex(id) + << " (as alternative) has incorrect miner transaction."); + bvc.m_verifivation_failed = true; + return false; + + } + + // FIXME: + // this brings up an interesting point: consider allowing to get block + // difficulty both by height OR by hash, not just height. + difficulty_type main_chain_cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); + if (alt_chain.size()) + { + bei.cumulative_difficulty = it_prev->second.cumulative_difficulty; + } + else + { + // passed-in block's previous block's cumulative difficulty, found on the main chain + bei.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id)); + } + bei.cumulative_difficulty += current_diff; + + // add block to alternate blocks storage, + // as well as the current "alt chain" container + auto i_res = m_alternative_chains.insert(blocks_ext_by_hash::value_type(id, bei)); + CHECK_AND_ASSERT_MES(i_res.second, false, "insertion of new alternative block returned as it already exist"); + alt_chain.push_back(i_res.first); + + // FIXME: is it even possible for a checkpoint to show up not on the main chain? + if(is_a_checkpoint) + { + //do reorganize! + LOG_PRINT_GREEN("###### REORGANIZE on height: " << alt_chain.front()->second.height << " of " << m_db->height() - 1 << + ", checkpoint is found in alternative chain on height " << bei.height, LOG_LEVEL_0); + + bool r = switch_to_alternative_blockchain(alt_chain, true); + + bvc.m_added_to_main_chain = r; + bvc.m_verifivation_failed = !r; + + return r; + } + else if(main_chain_cumulative_difficulty < bei.cumulative_difficulty) //check if difficulty bigger then in main chain + { + //do reorganize! + LOG_PRINT_GREEN("###### REORGANIZE on height: " + << alt_chain.front()->second.height << " of " << m_db->height() - 1 + << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height() - 1) + << std::endl << " alternative blockchain size: " << alt_chain.size() + << " with cum_difficulty " << bei.cumulative_difficulty, LOG_LEVEL_0 + ); + + bool r = switch_to_alternative_blockchain(alt_chain, false); + if(r) bvc.m_added_to_main_chain = true; + else bvc.m_verifivation_failed = true; + return r; + } + else + { + LOG_PRINT_BLUE("----- BLOCK ADDED AS ALTERNATIVE ON HEIGHT " << bei.height + << std::endl << "id:\t" << id + << std::endl << "PoW:\t" << proof_of_work + << std::endl << "difficulty:\t" << current_diff, LOG_LEVEL_0); + return true; + } + } + else + { + //block orphaned + bvc.m_marked_as_orphaned = true; + LOG_PRINT_RED_L1("Block recognized as orphaned and rejected, id = " << id); + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(start_offset > m_db->height()) + return false; + + if (!get_blocks(start_offset, count, blocks)) + { + return false; + } + + for(const block& blk : blocks) + { + std::list<crypto::hash> missed_ids; + get_transactions(blk.tx_hashes, txs, missed_ids); + CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(start_offset > m_db->height()) + return false; + + for(size_t i = start_offset; i < start_offset + count && i <= m_db->height();i++) + { + blocks.push_back(m_db->get_block_from_height(i)); + } + return true; +} +//------------------------------------------------------------------ +//TODO: This function *looks* like it won't need to be rewritten +// to use BlockchainDB, as it calls other functions that were, +// but it warrants some looking into later. +bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + rsp.current_blockchain_height = get_current_blockchain_height(); + std::list<block> blocks; + get_blocks(arg.blocks, blocks, rsp.missed_ids); + + BOOST_FOREACH(const auto& bl, blocks) + { + std::list<crypto::hash> missed_tx_id; + std::list<transaction> txs; + get_transactions(bl.tx_hashes, txs, rsp.missed_ids); + CHECK_AND_ASSERT_MES(!missed_tx_id.size(), false, "Internal error: has missed missed_tx_id.size()=" << missed_tx_id.size() + << std::endl << "for block id = " << get_block_hash(bl)); + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); + //pack block + e.block = t_serializable_object_to_blob(bl); + //pack transactions + BOOST_FOREACH(transaction& tx, txs) + e.txs.push_back(t_serializable_object_to_blob(tx)); + + } + //get another transactions, if need + std::list<transaction> txs; + get_transactions(arg.txs, txs, rsp.missed_ids); + //pack aside transactions + BOOST_FOREACH(const auto& tx, txs) + rsp.txs.push_back(t_serializable_object_to_blob(tx)); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_alternative_blocks(std::list<block>& blocks) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + BOOST_FOREACH(const auto& alt_bl, m_alternative_chains) + { + blocks.push_back(alt_bl.second.bl); + } + return true; +} +//------------------------------------------------------------------ +size_t Blockchain::get_alternative_blocks_count() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_alternative_chains.size(); +} +//------------------------------------------------------------------ +// This function adds the output specified by <amount, i> to the result_outs container +// unlocked and other such checks should be done by here. +void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); + oen.global_amount_index = i; + oen.out_key = m_db->get_output_key(amount, i); +} +//------------------------------------------------------------------ +// This function takes an RPC request for mixins and creates an RPC response +// with the requested mixins. +// TODO: figure out why this returns boolean / if we should be returning false +// in some cases +bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + srand(static_cast<unsigned int>(time(NULL))); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // for each amount that we need to get mixins for, get <n> random outputs + // from BlockchainDB where <n> is req.outs_count (number of mixins). + for (uint64_t amount : req.amounts) + { + // create outs_for_amount struct and populate amount field + COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); + result_outs.amount = amount; + + std::unordered_set<uint64_t> seen_indices; + + // if there aren't enough outputs to mix with (or just enough), + // use all of them. Eventually this should become impossible. + if (m_db->get_num_outputs(amount) <= req.outs_count) + { + for (uint64_t i = 0; i < m_db->get_num_outputs(amount); i++) + { + // get tx_hash, tx_out_index from DB + tx_out_index toi = m_db->get_output_tx_and_index(amount, i); + + // if tx is unlocked, add output to result_outs + if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) + { + add_out_to_get_random_outs(result_outs, amount, i); + } + + } + } + else + { + // while we still need more mixins + auto num_outs = m_db->get_num_outputs(amount); + while (result_outs.outs.size() < req.outs_count) + { + // if we've gone through every possible output, we've gotten all we can + if (seen_indices.size() == num_outs) + { + break; + } + + // get a random output index from the DB. If we've already seen it, + // return to the top of the loop and try again, otherwise add it to the + // list of output indices we've seen. + uint64_t i = m_db->get_random_output(amount); + if (seen_indices.count(i)) + { + continue; + } + seen_indices.emplace(i); + + // get tx_hash, tx_out_index from DB + tx_out_index toi = m_db->get_output_tx_and_index(amount, i); + + // if the output's transaction is unlocked, add the output's index to + // our list. + if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) + { + add_out_to_get_random_outs(result_outs, amount, i); + } + } + } + } + return true; +} +//------------------------------------------------------------------ +// This function takes a list of block hashes from another node +// on the network to find where the split point is between us and them. +// This is used to see what to send another node that needs to sync. +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // make sure the request includes at least the genesis block, otherwise + // how can we expect to sync from the client that the block list came from? + if(!qblock_ids.size() /*|| !req.m_total_height*/) + { + LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); + return false; + } + + // make sure that the last block in the request's block list matches + // the genesis block + auto gen_hash = m_db->get_block_hash_from_height(0); + if(qblock_ids.back() != gen_hash) + { + LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block missmatch: " << std::endl << "id: " + << qblock_ids.back() << ", " << std::endl << "expected: " << gen_hash + << "," << std::endl << " dropping connection"); + return false; + } + + // Find the first block the foreign chain has that we also have. + // Assume qblock_ids is in reverse-chronological order. + auto bl_it = qblock_ids.begin(); + uint64_t split_height = 0; + for(; bl_it != qblock_ids.end(); bl_it++) + { + try + { + split_height = m_db->get_block_height(*bl_it); + break; + } + catch (const BLOCK_DNE& e) + { + continue; + } + catch (const std::exception& e) + { + LOG_PRINT_L1("Non-critical error trying to find block by hash in BlockchainDB, hash: " << *bl_it); + return false; + } + } + + // this should be impossible, as we checked that we share the genesis block, + // but just in case... + if(bl_it == qblock_ids.end()) + { + LOG_PRINT_L1("Internal error handling connection, can't find split point"); + return false; + } + + // if split_height remains 0, we didn't have any but the genesis block in common + // which is only fine if the blocks just have the genesis block + if(split_height == 0 && qblock_ids.size() > 1) + { + LOG_ERROR("Ours and foreign blockchain have only genesis block in common... o.O"); + return false; + } + + //we start to put block ids INCLUDING last known id, just to make other side be sure + starter_offset = split_height; + return true; +} +//------------------------------------------------------------------ +uint64_t Blockchain::block_difficulty(uint64_t i) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + try + { + return m_db->get_block_difficulty(i); + } + catch (const BLOCK_DNE& e) + { + LOG_PRINT_L0("Attempted to get block difficulty for height above blockchain height"); + } + return 0; +} +//------------------------------------------------------------------ +template<class t_ids_container, class t_blocks_container, class t_missed_container> +bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (const auto& block_hash : block_ids) + { + try + { + blocks.push_back(m_db->get_block(block_hash)); + } + catch (const BLOCK_DNE& e) + { + missed_bs.push_back(block_hash); + } + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ +template<class t_ids_container, class t_tx_container, class t_missed_container> +bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (const auto& tx_hash : txs_ids) + { + try + { + txs.push_back(m_db->get_tx(tx_hash)); + } + catch (const TX_DNE& e) + { + missed_txs.push_back(tx_hash); + } + //FIXME: is this the correct way to handle this? + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ +void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::stringstream ss; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto h = m_db->height(); + if(start_index > h) + { + LOG_PRINT_L1("Wrong starter index set: " << start_index << ", expected max index " << h); + return; + } + + for(size_t i = start_index; i <= h && i != end_index; i++) + { + ss << "height " << i + << ", timestamp " << m_db->get_block_timestamp(i) + << ", cumul_dif " << m_db->get_block_cumulative_difficulty(i) + << ", size " << m_db->get_block_size(i) + << "\nid\t\t" << m_db->get_block_hash_from_height(i) + << "\ndifficulty\t\t" << m_db->get_block_difficulty(i) + << ", nonce " << m_db->get_block_from_height(i).nonce + << ", tx_count " << m_db->get_block_from_height(i).tx_hashes.size() + << std::endl; + } + LOG_PRINT_L1("Current blockchain:" << std::endl << ss.str()); + LOG_PRINT_L0("Blockchain printed with log level 1"); +} +//------------------------------------------------------------------ +void Blockchain::print_blockchain_index() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::stringstream ss; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto height = m_db->height(); + if (height != 0) + { + for(uint64_t i = 0; i <= height; i++) + { + ss << "height: " << i << ", hash: " << m_db->get_block_hash_from_height(i); + } + } + + LOG_PRINT_L0("Current blockchain index:" << std::endl + << ss.str() + ); +} +//------------------------------------------------------------------ +//TODO: remove this function and references to it +void Blockchain::print_blockchain_outs(const std::string& file) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + return; +} +//------------------------------------------------------------------ +// Find the split point between us and foreign blockchain and return +// (by reference) the most recent common block hash along with up to +// BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if we can't find the split point, return false + if(!find_blockchain_supplement(qblock_ids, resp.start_height)) + { + return false; + } + + resp.total_height = get_current_blockchain_height(); + size_t count = 0; + for(size_t i = resp.start_height; i < resp.total_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + { + resp.m_block_ids.push_back(m_db->get_block_hash_from_height(i)); + } + return true; +} +//------------------------------------------------------------------ +//FIXME: change argument to std::vector, low priority +// find split point between ours and foreign blockchain (or start at +// blockchain height <req_start_block>), and return up to max_count FULL +// blocks by reference. +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if a specific start height has been requested + if(req_start_block > 0) + { + // if requested height is higher than our chain, return false -- we can't help + if (req_start_block >= m_db->height()) + { + return false; + } + start_height = req_start_block; + } + else + { + if(!find_blockchain_supplement(qblock_ids, start_height)) + { + return false; + } + } + + total_height = get_current_blockchain_height(); + size_t count = 0; + for(size_t i = start_height; i < total_height && count < max_count; i++, count++) + { + blocks.resize(blocks.size()+1); + blocks.back().first = m_db->get_block_from_height(i); + std::list<crypto::hash> mis; + get_transactions(blocks.back().first.tx_hashes, blocks.back().second, mis); + CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + } + return true; +} +//------------------------------------------------------------------ +bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + block_extended_info bei = AUTO_VAL_INIT(bei); + bei.bl = bl; + return add_block_as_invalid(bei, h); +} +//------------------------------------------------------------------ +bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto i_res = m_invalid_blocks.insert(std::map<crypto::hash, block_extended_info>::value_type(h, bei)); + CHECK_AND_ASSERT_MES(i_res.second, false, "at insertion invalid by tx returned status existed"); + LOG_PRINT_L1("BLOCK ADDED AS INVALID: " << h << std::endl << ", prev_id=" << bei.bl.prev_id << ", m_invalid_blocks count=" << m_invalid_blocks.size()); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::have_block(const crypto::hash& id) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + if(m_db->block_exists(id)) + { + LOG_PRINT_L3("block exists in main chain"); + return true; + } + + if(m_alternative_chains.count(id)) + { + LOG_PRINT_L3("block found in m_alternative_chains"); + return true; + } + + if(m_invalid_blocks.count(id)) + { + LOG_PRINT_L3("block found in m_invalid_blocks"); + return true; + } + + return false; +} +//------------------------------------------------------------------ +bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + crypto::hash id = get_block_hash(bl); + return handle_block_to_main_chain(bl, id, bvc); +} +//------------------------------------------------------------------ +size_t Blockchain::get_total_transactions() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->get_tx_count(); +} +//------------------------------------------------------------------ +// This function checks each input in the transaction <tx> to make sure it +// has not been used already, and adds its key to the container <keys_this_block>. +// +// This container should be managed by the code that validates blocks so we don't +// have to store the used keys in a given block in the permanent storage only to +// remove them later if the block fails validation. +bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + struct add_transaction_input_visitor: public boost::static_visitor<bool> + { + key_images_container& m_spent_keys; + BlockchainDB* m_db; + add_transaction_input_visitor(key_images_container& spent_keys, BlockchainDB* db):m_spent_keys(spent_keys), m_db(db) + {} + bool operator()(const txin_to_key& in) const + { + const crypto::key_image& ki = in.k_image; + + // attempt to insert the newly-spent key into the container of + // keys spent this block. If this fails, the key was spent already + // in this block, return false to flag that a double spend was detected. + // + // if the insert into the block-wide spent keys container succeeds, + // check the blockchain-wide spent keys container and make sure the + // key wasn't used in another block already. + auto r = m_spent_keys.insert(ki); + if(!r.second || m_db->has_key_image(ki)) + { + //double spend detected + return false; + } + + // if no double-spend detected, return true + return true; + } + + bool operator()(const txin_gen& tx) const{return true;} + bool operator()(const txin_to_script& tx) const{return false;} + bool operator()(const txin_to_scripthash& tx) const{return false;} + }; + + for (const txin_v& in : tx.vin) + { + if(!boost::apply_visitor(add_transaction_input_visitor(keys_this_block, m_db), in)) + { + LOG_ERROR("Double spend detected!"); + return false; + } + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if (!m_db->tx_exists(tx_id)) + { + LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + return false; + } + + // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts + indexs = m_db->get_tx_amount_output_indices(tx_id); + CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); + + return true; +} +//------------------------------------------------------------------ +// This function overloads its sister function with +// an extra value (hash of highest block that holds an output used as input) +// as a return-by-reference. +bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + bool res = check_tx_inputs(tx, &max_used_block_height); + if(!res) return false; + CHECK_AND_ASSERT_MES(max_used_block_height < m_db->height(), false, "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db->height()); + max_used_block_id = m_db->get_block_hash_from_height(max_used_block_height); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + BOOST_FOREACH(const txin_v& in, tx.vin) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, in_to_key, true); + if(have_tx_keyimg_as_spent(in_to_key.k_image)) + return true; + } + return false; +} +//------------------------------------------------------------------ +// This function validates transaction inputs and their keys. Previously +// it also performed double spend checking, but that has been moved to its +// own function. +bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + size_t sig_index = 0; + if(pmax_used_block_height) + *pmax_used_block_height = 0; + + crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); + + for (const auto& txin : tx.vin) + { + // make sure output being spent is of type txin_to_key, rather than + // e.g. txin_gen, which is only used for miner transactions + CHECK_AND_ASSERT_MES(txin.type() == typeid(txin_to_key), false, "wrong type id in tx input at Blockchain::check_tx_inputs"); + const txin_to_key& in_to_key = boost::get<txin_to_key>(txin); + + // make sure tx output has key offset(s) (is signed to be used) + CHECK_AND_ASSERT_MES(in_to_key.key_offsets.size(), false, "empty in_to_key.key_offsets in transaction with id " << get_transaction_hash(tx)); + + // basically, make sure number of inputs == number of signatures + CHECK_AND_ASSERT_MES(sig_index < tx.signatures.size(), false, "wrong transaction: not signature entry for input with index= " << sig_index); + + // make sure that output being spent matches up correctly with the + // signature spending it. + if(!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[sig_index], pmax_used_block_height)) + { + LOG_PRINT_L1("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); + if (pmax_used_block_height) // a default value of NULL is used when called from Blockchain::handle_block_to_main_chain() + { + LOG_PRINT_L1(" *pmax_used_block_height: " << *pmax_used_block_height); + } + return false; + } + + sig_index++; + } + + return true; +} +//------------------------------------------------------------------ +// This function checks to see if a tx is unlocked. unlock_time is either +// a block index or a unix time. +bool Blockchain::is_tx_spendtime_unlocked(uint64_t unlock_time) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) + { + //interpret as block index + if(get_current_blockchain_height() + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + return true; + else + return false; + }else + { + //interpret as time + uint64_t current_time = static_cast<uint64_t>(time(NULL)); + if(current_time + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS >= unlock_time) + return true; + else + return false; + } + return false; +} +//------------------------------------------------------------------ +// This function locates all outputs associated with a given input (mixins) +// and validates that they exist and are usable. It also checks the ring +// signature for each input. +bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + struct outputs_visitor + { + std::vector<const crypto::public_key *>& m_p_output_keys; + std::vector<crypto::public_key >& m_output_keys; + const Blockchain& m_bch; + outputs_visitor(std::vector<crypto::public_key >& output_keys, std::vector<const crypto::public_key *>& p_output_keys, const Blockchain& bch) : m_output_keys(output_keys), m_p_output_keys(p_output_keys), m_bch(bch) + {} + bool handle_output(const transaction& tx, const tx_out& out) + { + //check tx unlock time + if(!m_bch.is_tx_spendtime_unlocked(tx.unlock_time)) + { + LOG_PRINT_L1("One of outputs for one of inputs has wrong tx.unlock_time = " << tx.unlock_time); + return false; + } + + if(out.target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Output has wrong type id, which=" << out.target.which()); + return false; + } + + m_output_keys.push_back(boost::get<txout_to_key>(out.target).key); + return true; + } + }; + + //check ring signature + std::vector<crypto::public_key> output_keys; + std::vector<const crypto::public_key *> p_output_keys; + outputs_visitor vi(output_keys, p_output_keys, *this); + if(!scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height)) + { + LOG_PRINT_L1("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size()); + return false; + } + + for (auto& k : output_keys) + { + p_output_keys.push_back(&k); + } + + if(txin.key_offsets.size() != output_keys.size()) + { + LOG_PRINT_L1("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size()); + return false; + } + CHECK_AND_ASSERT_MES(sig.size() == output_keys.size(), false, "internal error: tx signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size()); + if(m_is_in_checkpoint_zone) + return true; + return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, p_output_keys, sig.data()); +} +//------------------------------------------------------------------ +//TODO: Is this intended to do something else? Need to look into the todo there. +uint64_t Blockchain::get_adjusted_time() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + //TODO: add collecting median time + return time(NULL); +} +//------------------------------------------------------------------ +//TODO: revisit, has changed a bit on upstream +bool Blockchain::check_block_timestamp(std::vector<uint64_t>& timestamps, const block& b) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + uint64_t median_ts = epee::misc_utils::median(timestamps); + + if(b.timestamp < median_ts) + { + LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", less than median of last " << BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW << " blocks, " << median_ts); + return false; + } + + return true; +} +//------------------------------------------------------------------ +// This function grabs the timestamps from the most recent <n> blocks, +// where n = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. If there are not those many +// blocks in the blockchain, the timestap is assumed to be valid. If there +// are, this function returns: +// true if the block's timestamp is not less than the timestamp of the +// median of the selected blocks +// false otherwise +bool Blockchain::check_block_timestamp(const block& b) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) + { + LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); + return false; + } + + // if not enough blocks, no proper median yet, return true + if(m_db->height() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW + 1) + { + return true; + } + + std::vector<uint64_t> timestamps; + auto h = m_db->height(); + + // need most recent 60 blocks, get index of first of those + // using +1 because BlockchainDB::height() returns the index of the top block, + // not the size of the blockchain (0-indexed) + size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - 1; + for(;offset < h; ++offset) + { + timestamps.push_back(m_db->get_block_timestamp(offset)); + } + + return check_block_timestamp(timestamps, b); +} +//------------------------------------------------------------------ +// Needs to validate the block and acquire each transaction from the +// transaction mem_pool, then pass the block and transactions to +// m_db->add_block() +bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + + TIME_MEASURE_START(block_processing_time); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(bl.prev_id != get_tail_id()) + { + LOG_PRINT_L1("Block with id: " << id << std::endl + << "has wrong prev_id: " << bl.prev_id << std::endl + << "expected: " << get_tail_id()); + return false; + } + + // make sure block timestamp is not less than the median timestamp + // of a set number of the most recent blocks. + if(!check_block_timestamp(bl)) + { + LOG_PRINT_L1("Block with id: " << id << std::endl + << "has invalid timestamp: " << bl.timestamp); + bvc.m_verifivation_failed = true; + return false; + } + + //check proof of work + TIME_MEASURE_START(target_calculating_time); + + // get the target difficulty for the block. + // the calculation can overflow, among other failure cases, + // so we need to check the return type. + // FIXME: get_difficulty_for_next_block can also assert, look into + // changing this to throwing exceptions instead so we can clean up. + difficulty_type current_diffic = get_difficulty_for_next_block(); + CHECK_AND_ASSERT_MES(current_diffic, false, "!!!!!!!!! difficulty overhead !!!!!!!!!"); + + TIME_MEASURE_FINISH(target_calculating_time); + + TIME_MEASURE_START(longhash_calculating_time); + + crypto::hash proof_of_work = null_hash; + + // Formerly the code below contained an if loop with the following condition + // !m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height()) + // however, this caused the daemon to not bother checking PoW for blocks + // before checkpoints, which is very dangerous behaviour. We moved the PoW + // validation out of the next chunk of code to make sure that we correctly + // check PoW now. + // FIXME: height parameter is not used...should it be used or should it not + // be a parameter? + proof_of_work = get_block_longhash(bl, m_db->height()); + + // validate proof_of_work versus difficulty target + if(!check_hash(proof_of_work, current_diffic)) + { + LOG_PRINT_L1("Block with id: " << id << std::endl + << "does not have enough proof of work: " << proof_of_work << std::endl + << "unexpected difficulty: " << current_diffic ); + bvc.m_verifivation_failed = true; + return false; + } + + // If we're at a checkpoint, ensure that our hardcoded checkpoint hash + // is correct. + if(m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height())) + { + if(!m_checkpoints.check_block(get_current_blockchain_height(), id)) + { + LOG_ERROR("CHECKPOINT VALIDATION FAILED"); + bvc.m_verifivation_failed = true; + return false; + } + } + + TIME_MEASURE_FINISH(longhash_calculating_time); + + // sanity check basic miner tx properties + if(!prevalidate_miner_transaction(bl, m_db->height())) + { + LOG_PRINT_L1("Block with id: " << id + << " failed to pass prevalidation"); + bvc.m_verifivation_failed = true; + return false; + } + + size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); + size_t cumulative_block_size = coinbase_blob_size; + + std::vector<transaction> txs; + key_images_container keys; + + uint64_t fee_summary = 0; + + // Iterate over the block's transaction hashes, grabbing each + // from the tx_pool and validating them. Each is then added + // to txs. Keys spent in each are added to <keys> by the double spend check. + for (const crypto::hash& tx_id : bl.tx_hashes) + { + transaction tx; + size_t blob_size = 0; + uint64_t fee = 0; + + if (m_db->tx_exists(tx_id)) + { + LOG_PRINT_L1("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id); + bvc.m_verifivation_failed = true; + break; + } + + // get transaction with hash <tx_id> from tx_pool + if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + { + LOG_PRINT_L1("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); + bvc.m_verifivation_failed = true; + break; + } + + // add the transaction to the temp list of transactions, so we can either + // store the list of transactions all at once or return the ones we've + // taken from the tx_pool back to it if the block fails verification. + txs.push_back(tx); + + // validate that transaction inputs and the keys spending them are correct. + if(!check_tx_inputs(tx)) + { + LOG_PRINT_L1("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs."); + + //TODO: why is this done? make sure that keeping invalid blocks makes sense. + add_block_as_invalid(bl, id); + LOG_PRINT_L1("Block with id " << id << " added as invalid becouse of wrong inputs in transactions"); + bvc.m_verifivation_failed = true; + break; + } + + if (!check_for_double_spend(tx, keys)) + { + LOG_PRINT_L0("Double spend detected in transaction (id: " << tx_id); + bvc.m_verifivation_failed = true; + break; + } + + fee_summary += fee; + cumulative_block_size += blob_size; + } + + uint64_t base_reward = 0; + uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins)) + { + LOG_PRINT_L1("Block with id: " << id + << " has incorrect miner transaction"); + bvc.m_verifivation_failed = true; + } + + + block_extended_info bei = boost::value_initialized<block_extended_info>(); + size_t block_size; + difficulty_type cumulative_difficulty; + + // populate various metadata about the block to be stored alongside it. + block_size = cumulative_block_size; + cumulative_difficulty = current_diffic; + already_generated_coins = already_generated_coins + base_reward; + if(m_db->height()) + cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height() - 1); + + update_next_cumulative_size_limit(); + + TIME_MEASURE_FINISH(block_processing_time); + + uint64_t new_height = 0; + bool add_success = true; + if (!bvc.m_verifivation_failed) + { + try + { + new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + } + catch (const std::exception& e) + { + //TODO: figure out the best way to deal with this failure + LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + add_success = false; + } + } + + // if we failed for any reason to verify the block, return taken + // transactions to the tx_pool. + if (bvc.m_verifivation_failed || !add_success) + { + // return taken transactions to transaction pool + for (auto& tx : txs) + { + cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + if (!m_tx_pool.add_tx(tx, tvc, true)) + { + LOG_PRINT_L0("Failed to return taken transaction with hash: " << get_transaction_hash(tx) << " to tx_pool"); + } + } + return false; + } + + LOG_PRINT_L1("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id + << std::endl << "PoW:\t" << proof_of_work + << std::endl << "HEIGHT " << new_height << ", difficulty:\t" << current_diffic + << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) + << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size + << ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms"); + + bvc.m_added_to_main_chain = true; + + // appears to be a NOP *and* is called elsewhere. wat? + m_tx_pool.on_blockchain_inc(new_height, id); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::update_next_cumulative_size_limit() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::vector<size_t> sz; + get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + + uint64_t median = epee::misc_utils::median(sz); + if(median <= CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE) + median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE; + + m_current_block_cumul_sz_limit = median*2; + return true; +} +//------------------------------------------------------------------ +bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + //copy block here to let modify block.target + block bl = bl_; + crypto::hash id = get_block_hash(bl); + CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process + CRITICAL_REGION_LOCAL1(m_blockchain_lock); + if(have_block(id)) + { + LOG_PRINT_L3("block with id = " << id << " already exists"); + bvc.m_already_exists = true; + return false; + } + + //check that block refers to chain tail + if(!(bl.prev_id == get_tail_id())) + { + //chain switching or wrong block + bvc.m_added_to_main_chain = false; + return handle_alternative_block(bl, id, bvc); + //never relay alternative blocks + } + + return handle_block_to_main_chain(bl, id, bvc); +} +//------------------------------------------------------------------ +void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce) +{ + const auto& pts = points.get_points(); + + for (const auto& pt : pts) + { + // if the checkpoint is for a block we don't have yet, move on + if (pt.first >= m_db->height()) + { + continue; + } + + if (!points.check_block(pt.first, m_db->get_block_hash_from_height(pt.first))) + { + // if asked to enforce checkpoints, roll back to a couple of blocks before the checkpoint + if (enforce) + { + LOG_ERROR("Local blockchain failed to pass a checkpoint, rolling back!"); + std::list<block> empty; + rollback_blockchain_switching(empty, pt.first - 2); + } + else + { + LOG_ERROR("WARNING: local blockchain failed to pass a MoneroPulse checkpoint, and you could be on a fork. You should either sync up from scratch, OR download a fresh blockchain bootstrap, OR enable checkpoint enforcing with the --enforce-dns-checkpointing command-line option"); + } + } + } +} +//------------------------------------------------------------------ +// returns false if any of the checkpoints loading returns false. +// That should happen only if a checkpoint is added that conflicts +// with an existing checkpoint. +bool Blockchain::update_checkpoints(const std::string& file_path, bool check_dns) +{ + if (!cryptonote::load_checkpoints_from_json(m_checkpoints, file_path)) + { + return false; + } + + // if we're checking both dns and json, load checkpoints from dns. + // if we're not hard-enforcing dns checkpoints, handle accordingly + if (m_enforce_dns_checkpoints && check_dns) + { + if (!cryptonote::load_checkpoints_from_dns(m_checkpoints)) + { + return false; + } + } + else if (check_dns) + { + checkpoints dns_points; + cryptonote::load_checkpoints_from_dns(dns_points); + if (m_checkpoints.check_for_conflicts(dns_points)) + { + check_against_checkpoints(dns_points, false); + } + else + { + LOG_PRINT_L0("One or more checkpoints fetched from DNS conflicted with existing checkpoints!"); + } + } + + check_against_checkpoints(m_checkpoints, true); + + return true; +} +//------------------------------------------------------------------ +void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints) +{ + m_enforce_dns_checkpoints = enforce_checkpoints; +} diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h new file mode 100644 index 000000000..bc13901d2 --- /dev/null +++ b/src/cryptonote_core/blockchain.h @@ -0,0 +1,229 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once +#include <boost/serialization/serialization.hpp> +#include <boost/serialization/version.hpp> +#include <boost/serialization/list.hpp> +#include <boost/multi_index_container.hpp> +#include <boost/multi_index/global_fun.hpp> +#include <boost/multi_index/hashed_index.hpp> +#include <boost/multi_index/member.hpp> +#include <boost/foreach.hpp> +#include <atomic> + +#include "syncobj.h" +#include "string_tools.h" +#include "cryptonote_basic.h" +#include "common/util.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "difficulty.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "verification_context.h" +#include "crypto/hash.h" +#include "checkpoints.h" +#include "blockchain_db/blockchain_db.h" + +namespace cryptonote +{ + class tx_memory_pool; + + /************************************************************************/ + /* */ + /************************************************************************/ + class Blockchain + { + public: + struct transaction_chain_entry + { + transaction tx; + uint64_t m_keeper_block_height; + size_t m_blob_size; + std::vector<uint64_t> m_global_output_indexes; + }; + + struct block_extended_info + { + block bl; + uint64_t height; + size_t block_cumulative_size; + difficulty_type cumulative_difficulty; + uint64_t already_generated_coins; + }; + + Blockchain(tx_memory_pool& tx_pool); + + bool init(BlockchainDB* db, const bool testnet = false); + bool deinit(); + + void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } + + //bool push_new_block(); + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const; + bool get_alternative_blocks(std::list<block>& blocks) const; + size_t get_alternative_blocks_count() const; + crypto::hash get_block_id_by_height(uint64_t height) const; + bool get_block_by_hash(const crypto::hash &h, block &blk) const; + void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const; + + template<class archive_t> + void serialize(archive_t & ar, const unsigned int version); + + bool have_tx(const crypto::hash &id) const; + bool have_tx_keyimges_as_spent(const transaction &tx) const; + bool have_tx_keyimg_as_spent(const crypto::key_image &key_im) const; + + template<class visitor_t> + bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const; + + uint64_t get_current_blockchain_height() const; + crypto::hash get_tail_id() const; + crypto::hash get_tail_id(uint64_t& height) const; + difficulty_type get_difficulty_for_next_block() const; + bool add_new_block(const block& bl_, block_verification_context& bvc); + bool reset_and_set_genesis_block(const block& b); + bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce) const; + bool have_block(const crypto::hash& id) const; + size_t get_total_transactions() const; + bool get_short_chain_history(std::list<crypto::hash>& ids) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; + bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); + bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool store_blockchain(); + bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id) const; + uint64_t get_current_cumulative_blocksize_limit() const; + bool is_storing_blockchain()const{return m_is_blockchain_storing;} + uint64_t block_difficulty(uint64_t i) const; + + template<class t_ids_container, class t_blocks_container, class t_missed_container> + bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const; + + template<class t_ids_container, class t_tx_container, class t_missed_container> + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + + //debug functions + void print_blockchain(uint64_t start_index, uint64_t end_index); + void print_blockchain_index(); + void print_blockchain_outs(const std::string& file); + + void check_against_checkpoints(const checkpoints& points, bool enforce); + void set_enforce_dns_checkpoints(bool enforce); + bool update_checkpoints(const std::string& file_path, bool check_dns); + + BlockchainDB& get_db() + { + return *m_db; + } + + private: + typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index; + typedef std::unordered_map<crypto::hash, transaction_chain_entry> transactions_container; + typedef std::unordered_set<crypto::key_image> key_images_container; + typedef std::vector<block_extended_info> blocks_container; + typedef std::unordered_map<crypto::hash, block_extended_info> blocks_ext_by_hash; + typedef std::unordered_map<crypto::hash, block> blocks_by_hash; + typedef std::map<uint64_t, std::vector<std::pair<crypto::hash, size_t>>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction + + BlockchainDB* m_db; + + tx_memory_pool& m_tx_pool; + mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock + + // main chain + blocks_container m_blocks; // height -> block_extended_info + blocks_by_id_index m_blocks_index; // crypto::hash -> height + transactions_container m_transactions; + key_images_container m_spent_keys; + size_t m_current_block_cumul_sz_limit; + + + // all alternative chains + blocks_ext_by_hash m_alternative_chains; // crypto::hash -> block_extended_info + + // some invalid blocks + blocks_ext_by_hash m_invalid_blocks; // crypto::hash -> block_extended_info + outputs_container m_outputs; + + + checkpoints m_checkpoints; + std::atomic<bool> m_is_in_checkpoint_zone; + std::atomic<bool> m_is_blockchain_storing; + bool m_enforce_dns_checkpoints; + + bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain); + block pop_block_from_blockchain(); + bool purge_transaction_from_blockchain(const crypto::hash& tx_id); + bool purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check); + + bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); + bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); + bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const; + bool prevalidate_miner_transaction(const block& b, uint64_t height); + bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); + bool validate_transaction(const block& b, uint64_t height, const transaction& tx); + bool rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height); + bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); + bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes); + bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); + void get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; + void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; + bool is_tx_spendtime_unlocked(uint64_t unlock_time) const; + bool add_block_as_invalid(const block& bl, const crypto::hash& h); + bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); + bool check_block_timestamp(const block& b) const; + bool check_block_timestamp(std::vector<uint64_t>& timestamps, const block& b) const; + uint64_t get_adjusted_time() const; + bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); + bool update_next_cumulative_size_limit(); + + bool check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const; + }; + + + /************************************************************************/ + /* */ + /************************************************************************/ + + #define CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER 12 + + //------------------------------------------------------------------ + +} // namespace cryptonote + +BOOST_CLASS_VERSION(cryptonote::Blockchain, CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER) diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index 4cca878db..ee4263687 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -30,6 +30,7 @@ #include <algorithm> #include <cstdio> +#include <cmath> #include <boost/archive/binary_oarchive.hpp> #include <boost/archive/binary_iarchive.hpp> @@ -49,25 +50,27 @@ #include "crypto/hash.h" #include "cryptonote_core/checkpoints_create.h" //#include "serialization/json_archive.h" +#include "../../contrib/otshell_utils/utils.hpp" +#include "../../src/p2p/data_logger.hpp" using namespace cryptonote; DISABLE_VS_WARNINGS(4267) //------------------------------------------------------------------ -bool blockchain_storage::have_tx(const crypto::hash &id) +bool blockchain_storage::have_tx(const crypto::hash &id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_transactions.find(id) != m_transactions.end(); } //------------------------------------------------------------------ -bool blockchain_storage::have_tx_keyimg_as_spent(const crypto::key_image &key_im) +bool blockchain_storage::have_tx_keyimg_as_spent(const crypto::key_image &key_im) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_spent_keys.find(key_im) != m_spent_keys.end(); } //------------------------------------------------------------------ -transaction *blockchain_storage::get_tx(const crypto::hash &id) +const transaction *blockchain_storage::get_tx(const crypto::hash &id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_transactions.find(id); @@ -77,7 +80,7 @@ transaction *blockchain_storage::get_tx(const crypto::hash &id) return &it->second.tx; } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_current_blockchain_height() +uint64_t blockchain_storage::get_current_blockchain_height() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_blocks.size(); @@ -87,6 +90,7 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) { CRITICAL_REGION_LOCAL(m_blockchain_lock); m_config_folder = config_folder; + m_testnet = testnet; LOG_PRINT_L0("Loading blockchain..."); const std::string filename = m_config_folder + "/" CRYPTONOTE_BLOCKCHAINDATA_FILENAME; if(tools::unserialize_obj_from_file(*this, filename)) @@ -113,24 +117,14 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) else { LOG_PRINT_L0("Can't load blockchain storage from file, generating genesis block."); - block bl = boost::value_initialized<block>(); - block_verification_context bvc = boost::value_initialized<block_verification_context>(); - if (testnet) - { - generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else - { - generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); - } - add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed && bvc.m_added_to_main_chain, false, "Failed to add genesis block to blockchain"); + if (!store_genesis_block(testnet, true)) + return false; } if(!m_blocks.size()) { LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); - if (!store_genesis_block(testnet)) { + if (!store_genesis_block(testnet, false)) { return false; } } else { @@ -159,7 +153,7 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) return true; } //------------------------------------------------------------------ -bool blockchain_storage::store_genesis_block(bool testnet) { +bool blockchain_storage::store_genesis_block(bool testnet, bool check_added) { block bl = ::boost::value_initialized<block>(); block_verification_context bvc = boost::value_initialized<block_verification_context>(); @@ -173,7 +167,7 @@ bool blockchain_storage::store_genesis_block(bool testnet) { } add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); + CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed && (bvc.m_added_to_main_chain || !check_added), false, "Failed to add genesis block to blockchain"); return true; } //------------------------------------------------------------------ @@ -231,7 +225,7 @@ bool blockchain_storage::pop_block_from_blockchain() m_blocks_index.erase(bl_ind); //pop block from core m_blocks.pop_back(); - m_tx_pool.on_blockchain_dec(m_blocks.size()-1, get_tail_id()); + m_tx_pool->on_blockchain_dec(m_blocks.size()-1, get_tail_id()); return true; } //------------------------------------------------------------------ @@ -307,7 +301,7 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t if(!is_coinbase(tx)) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool r = m_tx_pool.add_tx(tx, tvc, true); + bool r = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES(r, false, "purge_block_data_from_blockchain: failed to add transaction to transaction pool"); } @@ -333,14 +327,14 @@ bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_ return res; } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_tail_id(uint64_t& height) +crypto::hash blockchain_storage::get_tail_id(uint64_t& height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); height = get_current_blockchain_height()-1; return get_tail_id(); } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_tail_id() +crypto::hash blockchain_storage::get_tail_id() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); crypto::hash id = null_hash; @@ -351,7 +345,7 @@ crypto::hash blockchain_storage::get_tail_id() return id; } //------------------------------------------------------------------ -bool blockchain_storage::get_short_chain_history(std::list<crypto::hash>& ids) +bool blockchain_storage::get_short_chain_history(std::list<crypto::hash>& ids) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); size_t i = 0; @@ -381,7 +375,7 @@ bool blockchain_storage::get_short_chain_history(std::list<crypto::hash>& ids) return true; } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) +crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(height >= m_blocks.size()) @@ -390,7 +384,7 @@ crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) return get_block_hash(m_blocks[height].bl); } //------------------------------------------------------------------ -bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) { +bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); // try to find block in main chain @@ -410,20 +404,20 @@ bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) { return false; } //------------------------------------------------------------------ -void blockchain_storage::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) { +void blockchain_storage::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); - BOOST_FOREACH(blocks_by_id_index::value_type &v, m_blocks_index) + BOOST_FOREACH(const blocks_by_id_index::value_type &v, m_blocks_index) main.push_back(v.first); - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_alternative_chains) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_alternative_chains) alt.push_back(v.first); - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_invalid_blocks) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_invalid_blocks) invalid.push_back(v.first); } //------------------------------------------------------------------ -difficulty_type blockchain_storage::get_difficulty_for_next_block() +difficulty_type blockchain_storage::get_difficulty_for_next_block() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector<uint64_t> timestamps; @@ -535,7 +529,7 @@ bool blockchain_storage::switch_to_alternative_blockchain(std::list<blocks_ext_b return true; } //------------------------------------------------------------------ -difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) +difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const { std::vector<uint64_t> timestamps; std::vector<difficulty_type> commulative_difficulties; @@ -580,7 +574,7 @@ difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(co return next_difficulty(timestamps, commulative_difficulties); } //------------------------------------------------------------------ -bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height) +bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height) const { CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); @@ -603,7 +597,7 @@ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t return true; } //------------------------------------------------------------------ -bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) +bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) const { //validate reward uint64_t money_in_use = 0; @@ -630,7 +624,7 @@ bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumul return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count) +bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); CHECK_AND_ASSERT_MES(from_height < m_blocks.size(), false, "Internal error: get_backward_blocks_sizes called with from_height=" << from_height << ", blockchain height = " << m_blocks.size()); @@ -642,7 +636,7 @@ bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vect return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) +bool blockchain_storage::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!m_blocks.size()) @@ -650,12 +644,12 @@ bool blockchain_storage::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t return get_backward_blocks_sizes(m_blocks.size() -1, sz, count); } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_current_comulative_blocksize_limit() +uint64_t blockchain_storage::get_current_cumulative_blocksize_limit() const { return m_current_block_cumul_sz_limit; } //------------------------------------------------------------------ -bool blockchain_storage::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) +bool blockchain_storage::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) const { size_t median_size; uint64_t already_generated_coins; @@ -676,16 +670,16 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad size_t txs_size; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { + if (!m_tx_pool->fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { return false; } #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) size_t real_txs_size = 0; uint64_t real_fee = 0; - CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); + CRITICAL_REGION_BEGIN(m_tx_pool->m_transactions_lock); BOOST_FOREACH(crypto::hash &cur_hash, b.tx_hashes) { - auto cur_res = m_tx_pool.m_transactions.find(cur_hash); - if (cur_res == m_tx_pool.m_transactions.end()) { + auto cur_res = m_tx_pool->m_transactions.find(cur_hash); + if (cur_res == m_tx_pool->m_transactions.end()) { LOG_ERROR("Creating block template: error: transaction not found"); continue; } @@ -774,7 +768,7 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad return false; } //------------------------------------------------------------------ -bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) +bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) const { if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) @@ -939,7 +933,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset >= m_blocks.size()) @@ -955,7 +949,7 @@ bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::li return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset >= m_blocks.size()) @@ -999,7 +993,7 @@ bool blockchain_storage::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_alternative_blocks(std::list<block>& blocks) +bool blockchain_storage::get_alternative_blocks(std::list<block>& blocks) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1010,21 +1004,21 @@ bool blockchain_storage::get_alternative_blocks(std::list<block>& blocks) return true; } //------------------------------------------------------------------ -size_t blockchain_storage::get_alternative_blocks_count() +size_t blockchain_storage::get_alternative_blocks_count() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_alternative_chains.size(); } //------------------------------------------------------------------ -bool blockchain_storage::add_out_to_get_random_outs(std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) +bool blockchain_storage::add_out_to_get_random_outs(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); - transactions_container::iterator tx_it = m_transactions.find(amount_outs[i].first); + transactions_container::const_iterator tx_it = m_transactions.find(amount_outs[i].first); CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "internal error: transaction with id " << amount_outs[i].first << ENDL << ", used in mounts global index for amount=" << amount << ": i=" << i << "not found in transactions index"); CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > amount_outs[i].second, false, "internal error: in global outs index, transaction out index=" << amount_outs[i].second << " more than transaction outputs = " << tx_it->second.tx.vout.size() << ", for tx id = " << amount_outs[i].first); - transaction& tx = tx_it->second.tx; + const transaction& tx = tx_it->second.tx; CHECK_AND_ASSERT_MES(tx.vout[amount_outs[i].second].target.type() == typeid(txout_to_key), false, "unknown tx out type"); //check if transaction is unlocked @@ -1037,7 +1031,7 @@ bool blockchain_storage::add_out_to_get_random_outs(std::vector<std::pair<crypto return true; } //------------------------------------------------------------------ -size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs) +size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!amount_outs.size()) @@ -1046,7 +1040,7 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair do { --i; - transactions_container::iterator it = m_transactions.find(amount_outs[i].first); + transactions_container::const_iterator it = m_transactions.find(amount_outs[i].first); CHECK_AND_ASSERT_MES(it != m_transactions.end(), 0, "internal error: failed to find transaction from outputs index with tx_id=" << amount_outs[i].first); if(it->second.m_keeper_block_height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW <= get_current_blockchain_height() ) return i+1; @@ -1054,7 +1048,7 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair return 0; } //------------------------------------------------------------------ -bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) +bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); BOOST_FOREACH(uint64_t amount, req.amounts) @@ -1067,7 +1061,7 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO LOG_PRINT_L1("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: not outs for amount " << amount << ", wallet should use some real outs when it lookup for some mix, so, at least one out for this amount should exist"); continue;//actually this is strange situation, wallet should use some real outs when it lookup for some mix, so, at least one out for this amount should exist } - std::vector<std::pair<crypto::hash, size_t> >& amount_outs = it->second; + const std::vector<std::pair<crypto::hash, size_t> >& amount_outs = it->second; //it is not good idea to use top fresh outs, because it increases possibility of transaction canceling on split //lets find upper bound of not fresh outs size_t up_index_limit = find_end_of_allowed_index(amount_outs); @@ -1078,7 +1072,13 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO size_t try_count = 0; for(uint64_t j = 0; j != req.outs_count && try_count < up_index_limit;) { - size_t i = crypto::rand<size_t>()%up_index_limit; + // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit + uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); + double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); + size_t i = (size_t)(frac*up_index_limit); + // just in case rounding up to 1 occurs after sqrt + if (i == up_index_limit) + --i; if(used.count(i)) continue; bool added = add_out_to_get_random_outs(amount_outs, result_outs, amount, i); @@ -1096,7 +1096,7 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO return true; } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) +bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1143,7 +1143,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash return true; } //------------------------------------------------------------------ -uint64_t blockchain_storage::block_difficulty(size_t i) +uint64_t blockchain_storage::block_difficulty(size_t i) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); CHECK_AND_ASSERT_MES(i < m_blocks.size(), false, "wrong block index i = " << i << " at blockchain_storage::block_difficulty()"); @@ -1153,6 +1153,31 @@ uint64_t blockchain_storage::block_difficulty(size_t i) return m_blocks[i].cumulative_difficulty - m_blocks[i-1].cumulative_difficulty; } //------------------------------------------------------------------ +double blockchain_storage::get_avg_block_size( size_t count) const +{ + if (count > get_current_blockchain_height()) return 500; + + double average = 0; + _dbg1_c("net/blksize", "HEIGHT: " << get_current_blockchain_height()); + _dbg1_c("net/blksize", "BLOCK ID BY HEIGHT: " << get_block_id_by_height(get_current_blockchain_height()) ); + _dbg1_c("net/blksize", "BLOCK TAIL ID: " << get_tail_id() ); + std::vector<size_t> size_vector; + + get_backward_blocks_sizes(get_current_blockchain_height() - count, size_vector, count); + + std::vector<size_t>::iterator it; + it = size_vector.begin(); + while (it != size_vector.end()) { + average += *it; + _dbg2_c("net/blksize", "VECTOR ELEMENT: " << (*it) ); + it++; + } + average = average / count; + _dbg1_c("net/blksize", "VECTOR SIZE: " << size_vector.size() << " average=" << average); + + return average; +} +//------------------------------------------------------------------ void blockchain_storage::print_blockchain(uint64_t start_index, uint64_t end_index) { std::stringstream ss; @@ -1206,7 +1231,7 @@ void blockchain_storage::print_blockchain_outs(const std::string& file) } } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) +bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!find_blockchain_supplement(qblock_ids, resp.start_height)) @@ -1219,7 +1244,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash return true; } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) +bool blockchain_storage::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(req_start_block > 0) { @@ -1258,7 +1283,7 @@ bool blockchain_storage::add_block_as_invalid(const block_extended_info& bei, co return true; } //------------------------------------------------------------------ -bool blockchain_storage::have_block(const crypto::hash& id) +bool blockchain_storage::have_block(const crypto::hash& id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_blocks_index.count(id)) @@ -1296,13 +1321,13 @@ bool blockchain_storage::push_transaction_to_global_outs_index(const transaction return true; } //------------------------------------------------------------------ -size_t blockchain_storage::get_total_transactions() +size_t blockchain_storage::get_total_transactions() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_transactions.size(); } //------------------------------------------------------------------ -bool blockchain_storage::get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys) +bool blockchain_storage::get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_outputs.find(amount); @@ -1338,7 +1363,7 @@ bool blockchain_storage::pop_transaction_from_global_index(const transaction& tx return true; } //------------------------------------------------------------------ -bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height) +bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t blob_size) { CRITICAL_REGION_LOCAL(m_blockchain_lock); struct add_transaction_input_visitor: public boost::static_visitor<bool> @@ -1377,6 +1402,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const } transaction_chain_entry ch_e; ch_e.m_keeper_block_height = bl_height; + ch_e.m_blob_size = blob_size; ch_e.tx = tx; auto i_r = m_transactions.insert(std::pair<crypto::hash, transaction_chain_entry>(tx_id, ch_e)); if(!i_r.second) @@ -1392,7 +1418,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) +bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_transactions.find(tx_id); @@ -1407,7 +1433,7 @@ bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std:: return true; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) +bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); bool res = check_tx_inputs(tx, &max_used_block_height); @@ -1417,7 +1443,7 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_us return true; } //------------------------------------------------------------------ -bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) +bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) const { BOOST_FOREACH(const txin_v& in, tx.vin) { @@ -1428,13 +1454,13 @@ bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) return false; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) +bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) const { crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); return check_tx_inputs(tx, tx_prefix_hash, pmax_used_block_height); } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) +bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) const { size_t sig_index = 0; if(pmax_used_block_height) @@ -1466,7 +1492,7 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::ha return true; } //------------------------------------------------------------------ -bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) +bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) const { if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { @@ -1487,15 +1513,15 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) return false; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height) +bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); struct outputs_visitor { std::vector<const crypto::public_key *>& m_results_collector; - blockchain_storage& m_bch; - outputs_visitor(std::vector<const crypto::public_key *>& results_collector, blockchain_storage& bch):m_results_collector(results_collector), m_bch(bch) + const blockchain_storage& m_bch; + outputs_visitor(std::vector<const crypto::public_key *>& results_collector, const blockchain_storage& bch):m_results_collector(results_collector), m_bch(bch) {} bool handle_output(const transaction& tx, const tx_out& out) { @@ -1537,13 +1563,13 @@ bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::h return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, output_keys, sig.data()); } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_adjusted_time() +uint64_t blockchain_storage::get_adjusted_time() const { //TODO: add collecting median time return time(NULL); } //------------------------------------------------------------------ -bool blockchain_storage::check_block_timestamp_main(const block& b) +bool blockchain_storage::check_block_timestamp_main(const block& b) const { if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) { @@ -1559,7 +1585,7 @@ bool blockchain_storage::check_block_timestamp_main(const block& b) return check_block_timestamp(std::move(timestamps), b); } //------------------------------------------------------------------ -bool blockchain_storage::check_block_timestamp(std::vector<uint64_t> timestamps, const block& b) +bool blockchain_storage::check_block_timestamp(std::vector<uint64_t> timestamps, const block& b) const { if(timestamps.size() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) return true; @@ -1642,10 +1668,12 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt bvc.m_verifivation_failed = true; return false; } - size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); + crypto::hash coinbase_hash = null_hash; + size_t coinbase_blob_size = 0; + get_transaction_hash(bl.miner_tx, coinbase_hash, coinbase_blob_size); size_t cumulative_block_size = coinbase_blob_size; //process transactions - if(!add_transaction_from_block(bl.miner_tx, get_transaction_hash(bl.miner_tx), id, get_current_blockchain_height())) + if(!add_transaction_from_block(bl.miner_tx, coinbase_hash, id, get_current_blockchain_height(), coinbase_blob_size)) { LOG_PRINT_L1("Block with id: " << id << " failed to add transaction to blockchain storage"); bvc.m_verifivation_failed = true; @@ -1658,7 +1686,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt transaction tx; size_t blob_size = 0; uint64_t fee = 0; - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + if(!m_tx_pool->take_tx(tx_id, tx, blob_size, fee)) { LOG_PRINT_L1("Block with id: " << id << "has at least one unknown transaction with id: " << tx_id); purge_block_data_from_blockchain(bl, tx_processed_count); @@ -1670,7 +1698,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt { LOG_PRINT_L1("Block with id: " << id << "has at least one transaction (id: " << tx_id << ") with wrong inputs."); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true); + bool add_res = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES2(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); purge_block_data_from_blockchain(bl, tx_processed_count); add_block_as_invalid(bl, id); @@ -1679,11 +1707,11 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } - if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height())) + if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height(), blob_size)) { LOG_PRINT_L1("Block with id: " << id << " failed to add transaction to blockchain storage"); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true); + bool add_res = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES2(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); purge_block_data_from_blockchain(bl, tx_processed_count); bvc.m_verifivation_failed = true; @@ -1709,7 +1737,14 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt bei.bl = bl; bei.block_cumulative_size = cumulative_block_size; bei.cumulative_difficulty = current_diffic; - bei.already_generated_coins = already_generated_coins + base_reward; + + // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of + // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins + // at MONEY_SUPPLY. already_generated_coins is only used to compute the block subsidy and MONEY_SUPPLY yields a + // subsidy of 0 under the base formula and therefore the minimum subsidy >0 in the tail state. + + bei.already_generated_coins = base_reward < (MONEY_SUPPLY-already_generated_coins) ? already_generated_coins + base_reward : MONEY_SUPPLY; + if(m_blocks.size()) bei.cumulative_difficulty += m_blocks.back().cumulative_difficulty; @@ -1734,11 +1769,13 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms"); + epee::net_utils::data_logger::get_instance().add_data("blockchain_processing_time", block_processing_time); + bvc.m_added_to_main_chain = true; /*if(!m_orphanes_reorganize_in_work) review_orphaned_blocks_with_new_block_id(id, true);*/ - m_tx_pool.on_blockchain_inc(bei.height, id); + m_tx_pool->on_blockchain_inc(bei.height, id); //LOG_PRINT_L0("BLOCK: " << ENDL << "" << dump_obj_as_json(bei.bl)); return true; } @@ -1761,7 +1798,7 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont //copy block here to let modify block.target block bl = bl_; crypto::hash id = get_block_hash(bl); - CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process + CRITICAL_REGION_LOCAL(*m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process CRITICAL_REGION_LOCAL1(m_blockchain_lock); if(have_block(id)) { @@ -1782,7 +1819,7 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -void blockchain_storage::check_against_checkpoints(checkpoints& points, bool enforce) +void blockchain_storage::check_against_checkpoints(const checkpoints& points, bool enforce) { const auto& pts = points.get_points(); @@ -1833,7 +1870,7 @@ bool blockchain_storage::update_checkpoints(const std::string& file_path, bool c else if (check_dns) { checkpoints dns_points; - cryptonote::load_checkpoints_from_dns(dns_points); + cryptonote::load_checkpoints_from_dns(dns_points, m_testnet); if (m_checkpoints.check_for_conflicts(dns_points)) { check_against_checkpoints(dns_points, false); diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 08c007a4a..50a62a428 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -78,7 +78,7 @@ namespace cryptonote uint64_t already_generated_coins; }; - blockchain_storage(tx_memory_pool& tx_pool):m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) + blockchain_storage(tx_memory_pool* tx_pool):m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) {}; bool init() { return init(tools::get_default_data_dir(), true); } @@ -88,55 +88,56 @@ namespace cryptonote void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } //bool push_new_block(); - bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs); - bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks); - bool get_alternative_blocks(std::list<block>& blocks); - size_t get_alternative_blocks_count(); - crypto::hash get_block_id_by_height(uint64_t height); - bool get_block_by_hash(const crypto::hash &h, block &blk); - void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid); + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const; + bool get_alternative_blocks(std::list<block>& blocks) const; + size_t get_alternative_blocks_count() const; + crypto::hash get_block_id_by_height(uint64_t height) const; + bool get_block_by_hash(const crypto::hash &h, block &blk) const; + void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const; template<class archive_t> void serialize(archive_t & ar, const unsigned int version); - bool have_tx(const crypto::hash &id); - bool have_tx_keyimges_as_spent(const transaction &tx); - bool have_tx_keyimg_as_spent(const crypto::key_image &key_im); - transaction *get_tx(const crypto::hash &id); + bool have_tx(const crypto::hash &id) const; + bool have_tx_keyimges_as_spent(const transaction &tx) const; + bool have_tx_keyimg_as_spent(const crypto::key_image &key_im) const; + const transaction *get_tx(const crypto::hash &id) const; template<class visitor_t> - bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL); + bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const; - uint64_t get_current_blockchain_height(); - crypto::hash get_tail_id(); - crypto::hash get_tail_id(uint64_t& height); - difficulty_type get_difficulty_for_next_block(); + uint64_t get_current_blockchain_height() const; + crypto::hash get_tail_id() const; + crypto::hash get_tail_id(uint64_t& height) const; + difficulty_type get_difficulty_for_next_block() const; bool add_new_block(const block& bl_, block_verification_context& bvc); bool reset_and_set_genesis_block(const block& b); - bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce); - bool have_block(const crypto::hash& id); - size_t get_total_transactions(); - bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys); - bool get_short_chain_history(std::list<crypto::hash>& ids); - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp); - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset); - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); + bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce) const; + bool have_block(const crypto::hash& id) const; + size_t get_total_transactions() const; + bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys) const; + bool get_short_chain_history(std::list<crypto::hash>& ids) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count); - bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; + bool get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; bool store_blockchain(); - bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height = NULL); - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id); - uint64_t get_current_comulative_blocksize_limit(); - bool is_storing_blockchain(){return m_is_blockchain_storing;} - uint64_t block_difficulty(size_t i); + bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id) const; + uint64_t get_current_cumulative_blocksize_limit() const; + bool is_storing_blockchain()const{return m_is_blockchain_storing;} + uint64_t block_difficulty(size_t i) const; + double get_avg_block_size( size_t count) const; template<class t_ids_container, class t_blocks_container, class t_missed_container> - bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) + bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -156,7 +157,7 @@ namespace cryptonote } template<class t_ids_container, class t_tx_container, class t_missed_container> - bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -166,7 +167,7 @@ namespace cryptonote if(it == m_transactions.end()) { transaction tx; - if(!m_tx_pool.get_transaction(tx_id, tx)) + if(!m_tx_pool->get_transaction(tx_id, tx)) missed_txs.push_back(tx_id); else txs.push_back(tx); @@ -180,10 +181,15 @@ namespace cryptonote void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); void print_blockchain_outs(const std::string& file); - void check_against_checkpoints(checkpoints& points, bool enforce); + void check_against_checkpoints(const checkpoints& points, bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); void set_enforce_dns_checkpoints(bool enforce_checkpoints); + block get_block(uint64_t height) const { return m_blocks[height].bl; } + size_t get_block_size(uint64_t height) const { return m_blocks[height].block_cumulative_size; } + difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return m_blocks[height].cumulative_difficulty; } + uint64_t get_block_coins_generated(uint64_t height) const { return m_blocks[height].already_generated_coins; } + private: typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index; typedef std::unordered_map<crypto::hash, transaction_chain_entry> transactions_container; @@ -193,8 +199,8 @@ namespace cryptonote typedef std::unordered_map<crypto::hash, block> blocks_by_hash; typedef std::map<uint64_t, std::vector<std::pair<crypto::hash, size_t>>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction - tx_memory_pool& m_tx_pool; - epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock + tx_memory_pool* m_tx_pool; + mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock // main chain blocks_container m_blocks; // height -> block_extended_info @@ -218,6 +224,7 @@ namespace cryptonote std::atomic<bool> m_is_blockchain_storing; bool m_enforce_dns_checkpoints; + bool m_testnet; bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain); bool pop_block_from_blockchain(); @@ -228,26 +235,26 @@ namespace cryptonote bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); - difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei); - bool prevalidate_miner_transaction(const block& b, uint64_t height); - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); - bool validate_transaction(const block& b, uint64_t height, const transaction& tx); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const; + bool prevalidate_miner_transaction(const block& b, uint64_t height) const; + bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) const; + bool validate_transaction(const block& b, uint64_t height, const transaction& tx) const; bool rollback_blockchain_switching(std::list<block>& original_chain, size_t rollback_height); - bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); + bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t blob_size); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); - bool get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count); - bool add_out_to_get_random_outs(std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); - bool is_tx_spendtime_unlocked(uint64_t unlock_time); + bool get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; + bool add_out_to_get_random_outs(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; + bool is_tx_spendtime_unlocked(uint64_t unlock_time) const; bool add_block_as_invalid(const block& bl, const crypto::hash& h); bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); - size_t find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs); - bool check_block_timestamp_main(const block& b); - bool check_block_timestamp(std::vector<uint64_t> timestamps, const block& b); - uint64_t get_adjusted_time(); - bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); + size_t find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs) const; + bool check_block_timestamp_main(const block& b) const; + bool check_block_timestamp(std::vector<uint64_t> timestamps, const block& b) const; + uint64_t get_adjusted_time() const; + bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps) const; bool update_next_comulative_size_limit(); - bool store_genesis_block(bool testnet); + bool store_genesis_block(bool testnet, bool check_added = false); }; @@ -315,7 +322,7 @@ namespace cryptonote //------------------------------------------------------------------ template<class visitor_t> - bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) + bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_outputs.find(tx_in_to_key.amount); @@ -325,7 +332,7 @@ namespace cryptonote std::vector<uint64_t> absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); - std::vector<std::pair<crypto::hash, size_t> >& amount_outs_vec = it->second; + const std::vector<std::pair<crypto::hash, size_t> >& amount_outs_vec = it->second; size_t count = 0; BOOST_FOREACH(uint64_t i, absolute_offsets) { @@ -334,7 +341,7 @@ namespace cryptonote LOG_PRINT_L0("Wrong index in transaction inputs: " << i << ", expected maximum " << amount_outs_vec.size() - 1); return false; } - transactions_container::iterator tx_it = m_transactions.find(amount_outs_vec[i].first); + transactions_container::const_iterator tx_it = m_transactions.find(amount_outs_vec[i].first); CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "Wrong transaction id in output indexes: " << epee::string_tools::pod_to_hex(amount_outs_vec[i].first)); CHECK_AND_ASSERT_MES(amount_outs_vec[i].second < tx_it->second.tx.vout.size(), false, "Wrong index in transaction outputs: " << amount_outs_vec[i].second << ", expected less then " << tx_it->second.tx.vout.size()); diff --git a/src/cryptonote_core/blockchain_storage_boost_serialization.h b/src/cryptonote_core/blockchain_storage_boost_serialization.h index c64ce8e0c..28c06ece1 100644 --- a/src/cryptonote_core/blockchain_storage_boost_serialization.h +++ b/src/cryptonote_core/blockchain_storage_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/checkpoints.cpp b/src/cryptonote_core/checkpoints.cpp index 6a86302b6..e4223afb5 100644 --- a/src/cryptonote_core/checkpoints.cpp +++ b/src/cryptonote_core/checkpoints.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -84,6 +84,10 @@ namespace cryptonote return check_block(height, h, ignored); } //--------------------------------------------------------------------------- + // this basically says if the blockchain is smaller than the first + // checkpoint then alternate blocks are allowed. Alternatively, if the + // last checkpoint *before* the end of the current chain is also before + // the block to be added, then this is fine. bool checkpoints::is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const { if (0 == block_height) @@ -99,7 +103,7 @@ namespace cryptonote return checkpoint_height < block_height; } //--------------------------------------------------------------------------- - uint64_t checkpoints::get_max_height() + uint64_t checkpoints::get_max_height() const { std::map< uint64_t, crypto::hash >::const_iterator highest = std::max_element( m_points.begin(), m_points.end(), @@ -108,18 +112,18 @@ namespace cryptonote return highest->first; } //--------------------------------------------------------------------------- - const std::map<uint64_t, crypto::hash>& checkpoints::get_points() + const std::map<uint64_t, crypto::hash>& checkpoints::get_points() const { return m_points; } - bool checkpoints::check_for_conflicts(checkpoints& other) + bool checkpoints::check_for_conflicts(const checkpoints& other) const { for (auto& pt : other.get_points()) { if (m_points.count(pt.first)) { - CHECK_AND_ASSERT_MES(pt.second == m_points[pt.first], false, "Checkpoint at given height already exists, and hash for new checkpoint was different!"); + CHECK_AND_ASSERT_MES(pt.second == m_points.at(pt.first), false, "Checkpoint at given height already exists, and hash for new checkpoint was different!"); } } return true; diff --git a/src/cryptonote_core/checkpoints.h b/src/cryptonote_core/checkpoints.h index 132917228..55d765b71 100644 --- a/src/cryptonote_core/checkpoints.h +++ b/src/cryptonote_core/checkpoints.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -45,9 +45,9 @@ namespace cryptonote bool check_block(uint64_t height, const crypto::hash& h) const; bool check_block(uint64_t height, const crypto::hash& h, bool& is_a_checkpoint) const; bool is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const; - uint64_t get_max_height(); - const std::map<uint64_t, crypto::hash>& get_points(); - bool check_for_conflicts(checkpoints& other); + uint64_t get_max_height() const; + const std::map<uint64_t, crypto::hash>& get_points() const; + bool check_for_conflicts(const checkpoints& other) const; private: std::map<uint64_t, crypto::hash> m_points; }; diff --git a/src/cryptonote_core/checkpoints_create.cpp b/src/cryptonote_core/checkpoints_create.cpp index 9ad0a27e8..6f22e596d 100644 --- a/src/cryptonote_core/checkpoints_create.cpp +++ b/src/cryptonote_core/checkpoints_create.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -35,6 +35,30 @@ #include <random> #include "storages/portable_storage_template_helper.h" // epee json include +namespace +{ + bool dns_records_match(const std::vector<std::string>& a, const std::vector<std::string>& b) + { + if (a.size() != b.size()) return false; + + for (const auto& record_in_a : a) + { + bool ok = false; + for (const auto& record_in_b : b) + { + if (record_in_a == record_in_b) + { + ok = true; + break; + } + } + if (!ok) return false; + } + + return true; + } +} // anonymous namespace + namespace cryptonote { @@ -76,6 +100,7 @@ bool create_checkpoints(cryptonote::checkpoints& checkpoints) ADD_CHECKPOINT(231350, "b5add137199b820e1ea26640e5c3e121fd85faa86a1e39cf7e6cc097bdeb1131"); ADD_CHECKPOINT(232150, "955de8e6b6508af2c24f7334f97beeea651d78e9ade3ab18fec3763be3201aa8"); ADD_CHECKPOINT(249380, "654fb0a81ce3e5caf7e3264a70f447d4bd07586c08fa50f6638cc54da0a52b2d"); + ADD_CHECKPOINT(460000, "75037a7aed3e765db96c75bcf908f59d690a5f3390baebb9edeafd336a1c4831"); return true; } @@ -112,7 +137,7 @@ bool load_checkpoints_from_json(cryptonote::checkpoints& checkpoints, std::strin return true; } -bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints) +bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testnet) { // All four MoneroPulse domains have DNSSEC on and valid static const std::vector<std::string> dns_urls = { "checkpoints.moneropulse.se" @@ -120,43 +145,97 @@ bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints) , "checkpoints.moneropulse.net" , "checkpoints.moneropulse.co" }; - bool avail, valid; - std::vector<std::string> records; + + static const std::vector<std::string> testnet_dns_urls = { "testpoints.moneropulse.se" + , "testpoints.moneropulse.org" + , "testpoints.moneropulse.net" + , "testpoints.moneropulse.co" + }; + + std::vector<std::vector<std::string> > records; + records.resize(dns_urls.size()); std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<int> dis(0, dns_urls.size() - 1); size_t first_index = dis(gen); + bool avail, valid; size_t cur_index = first_index; do { - records = tools::DNSResolver::instance().get_txt_record(dns_urls[cur_index], avail, valid); - if (records.size() == 0 || (avail && !valid)) + std::string url; + if (testnet) + { + url = testnet_dns_urls[cur_index]; + } + else + { + url = dns_urls[cur_index]; + } + + records[cur_index] = tools::DNSResolver::instance().get_txt_record(url, avail, valid); + if (!avail) + { + LOG_PRINT_L2("DNSSEC not available for checkpoint update at URL: " << url << ", skipping."); + } + if (!valid) + { + LOG_PRINT_L2("DNSSEC validation failed for checkpoint update at URL: " << url << ", skipping."); + } + + if (records[cur_index].size() == 0 || !avail || !valid) { cur_index++; if (cur_index == dns_urls.size()) { cur_index = 0; } + records[cur_index].clear(); continue; } break; } while (cur_index != first_index); - if (records.size() == 0) + size_t num_valid_records = 0; + + for( const auto& record_set : records) + { + if (record_set.size() != 0) + { + num_valid_records++; + } + } + + if (num_valid_records < 2) { - LOG_PRINT_L1("Fetching MoneroPulse checkpoints failed, no TXT records available."); + LOG_PRINT_L0("WARNING: no two valid MoneroPulse DNS checkpoint records were received"); return true; } - if (avail && !valid) + int good_records_index = -1; + for (size_t i = 0; i < records.size() - 1; ++i) + { + if (records[i].size() == 0) continue; + + for (size_t j = i + 1; j < records.size(); ++j) + { + if (dns_records_match(records[i], records[j])) + { + good_records_index = i; + break; + } + } + if (good_records_index >= 0) break; + } + + if (good_records_index < 0) { - LOG_PRINT_L0("WARNING: MoneroPulse failed DNSSEC validation and/or returned no records"); + LOG_PRINT_L0("WARNING: no two MoneroPulse DNS checkpoint records matched"); return true; } - for (auto& record : records) + for (auto& record : records[good_records_index]) { auto pos = record.find(":"); if (pos != std::string::npos) diff --git a/src/cryptonote_core/checkpoints_create.h b/src/cryptonote_core/checkpoints_create.h index 92a970e9f..8422e2b33 100644 --- a/src/cryptonote_core/checkpoints_create.h +++ b/src/cryptonote_core/checkpoints_create.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -42,7 +42,7 @@ namespace cryptonote bool create_checkpoints(cryptonote::checkpoints& checkpoints); bool load_checkpoints_from_json(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath); - bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints); + bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testnet = false); bool load_new_checkpoints(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath); } // namespace cryptonote diff --git a/src/cryptonote_core/connection_context.h b/src/cryptonote_core/connection_context.h index 6574e92e6..7d40bc2e9 100644 --- a/src/cryptonote_core/connection_context.h +++ b/src/cryptonote_core/connection_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_basic.h b/src/cryptonote_core/cryptonote_basic.h index dfdfa03e5..2be76c0de 100644 --- a/src/cryptonote_core/cryptonote_basic.h +++ b/src/cryptonote_core/cryptonote_basic.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -50,6 +50,8 @@ #include "misc_language.h" #include "tx_extra.h" +#define DB_MEMORY 1 +#define DB_LMDB 2 namespace cryptonote { diff --git a/src/cryptonote_core/cryptonote_basic_impl.cpp b/src/cryptonote_core/cryptonote_basic_impl.cpp index 9cec666f2..f2d862e57 100644 --- a/src/cryptonote_core/cryptonote_basic_impl.cpp +++ b/src/cryptonote_core/cryptonote_basic_impl.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -60,6 +60,10 @@ namespace cryptonote { //----------------------------------------------------------------------------------------------- bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward) { uint64_t base_reward = (MONEY_SUPPLY - already_generated_coins) >> EMISSION_SPEED_FACTOR; + if (base_reward < FINAL_SUBSIDY_PER_MINUTE) + { + base_reward = FINAL_SUBSIDY_PER_MINUTE; + } //make it soft if (median_size < CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE) { diff --git a/src/cryptonote_core/cryptonote_basic_impl.h b/src/cryptonote_core/cryptonote_basic_impl.h index ddef677cb..5c773c4e1 100644 --- a/src/cryptonote_core/cryptonote_basic_impl.h +++ b/src/cryptonote_core/cryptonote_basic_impl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_boost_serialization.h b/src/cryptonote_core/cryptonote_boost_serialization.h index e3ccb7c32..17face4bd 100644 --- a/src/cryptonote_core/cryptonote_boost_serialization.h +++ b/src/cryptonote_core/cryptonote_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index d9d8a7faf..a5517e011 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -42,6 +42,13 @@ using namespace epee; #include "cryptonote_format_utils.h" #include "misc_language.h" #include <csignal> +#include "daemon/command_line_args.h" +#include "cryptonote_core/checkpoints_create.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#ifndef STATICLIB +#include "blockchain_db/berkeleydb/db_bdb.h" +#endif DISABLE_VS_WARNINGS(4355) @@ -51,7 +58,11 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- core::core(i_cryptonote_protocol* pprotocol): m_mempool(m_blockchain_storage), +#if BLOCKCHAIN_DB == DB_LMDB m_blockchain_storage(m_mempool), +#else + m_blockchain_storage(&m_mempool), +#endif m_miner(this), m_miner_address(boost::value_initialized<account_public_address>()), m_starter_message_showed(false), @@ -87,6 +98,10 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::update_checkpoints() { + if (m_testnet) return true; + + if (m_checkpoints_updating.test_and_set()) return true; + bool res = true; if (time(NULL) - m_last_dns_checkpoints_update >= 3600) { @@ -100,6 +115,8 @@ namespace cryptonote m_last_json_checkpoints_update = time(NULL); } + m_checkpoints_updating.clear(); + // if anything fishy happened getting new checkpoints, bring down the house if (!res) { @@ -108,14 +125,46 @@ namespace cryptonote return res; } //----------------------------------------------------------------------------------- + void core::stop() + { + graceful_exit(); + } + //----------------------------------------------------------------------------------- void core::init_options(boost::program_options::options_description& /*desc*/) { } //----------------------------------------------------------------------------------------------- - bool core::handle_command_line(const boost::program_options::variables_map& vm, bool testnet) + bool core::handle_command_line(const boost::program_options::variables_map& vm) { - auto data_dir_arg = testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + m_testnet = command_line::get_arg(vm, daemon_args::arg_testnet_on); + + auto data_dir_arg = m_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; m_config_folder = command_line::get_arg(vm, data_dir_arg); + + auto data_dir = boost::filesystem::path(m_config_folder); + + if (!m_testnet) + { + cryptonote::checkpoints checkpoints; + if (!cryptonote::create_checkpoints(checkpoints)) + { + throw std::runtime_error("Failed to initialize checkpoints"); + } + set_checkpoints(std::move(checkpoints)); + + boost::filesystem::path json(JSON_HASH_FILE_NAME); + boost::filesystem::path checkpoint_json_hashfile_fullpath = data_dir / json; + + set_checkpoints_file_path(checkpoint_json_hashfile_fullpath.string()); + } + + + set_enforce_dns_checkpoints(command_line::get_arg(vm, daemon_args::arg_dns_checkpoints)); + test_drop_download_height(command_line::get_arg(vm, command_line::arg_test_drop_download_height)); + + if (command_line::get_arg(vm, command_line::arg_test_drop_download) == true) + test_drop_download(); + return true; } //----------------------------------------------------------------------------------------------- @@ -154,21 +203,64 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, bool testnet) + bool core::init(const boost::program_options::variables_map& vm) { - bool r = handle_command_line(vm, testnet); + bool r = handle_command_line(vm); r = m_mempool.init(m_config_folder); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); - r = m_blockchain_storage.init(m_config_folder, testnet); +#if BLOCKCHAIN_DB == DB_LMDB + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + { + db = new BlockchainLMDB(); + } + else if (db_type == "berkeley") + { +#ifndef STATICLIB + db = new BlockchainBDB(); +#else + LOG_ERROR("BlockchainBDB not supported on STATIC builds"); + return false; +#endif + } + else + { + LOG_ERROR("Attempted to use non-existant database type"); + return false; + } + + boost::filesystem::path folder(m_config_folder); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename); + } + catch (const DB_ERROR& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return false; + } + + r = m_blockchain_storage.init(db, m_testnet); +#else + r = m_blockchain_storage.init(m_config_folder, m_testnet); +#endif CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); // load json & DNS checkpoints, and verify them // with respect to what blocks we already have CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); - r = m_miner.init(vm, testnet); + r = m_miner.init(vm, m_testnet); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); return load_state_data(); @@ -187,12 +279,51 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::deinit() { - m_miner.stop(); - m_mempool.deinit(); - m_blockchain_storage.deinit(); + m_miner.stop(); + m_mempool.deinit(); + if (!m_fast_exit) + { + m_blockchain_storage.deinit(); + } return true; } //----------------------------------------------------------------------------------------------- + void core::set_fast_exit() + { + m_fast_exit = true; + } + //----------------------------------------------------------------------------------------------- + bool core::get_fast_exit() + { + return m_fast_exit; + } + //----------------------------------------------------------------------------------------------- + void core::test_drop_download() + { + m_test_drop_download = false; + } + //----------------------------------------------------------------------------------------------- + void core::test_drop_download_height(uint64_t height) + { + m_test_drop_download_height = height; + } + //----------------------------------------------------------------------------------------------- + bool core::get_test_drop_download() + { + return m_test_drop_download; + } + //----------------------------------------------------------------------------------------------- + bool core::get_test_drop_download_height() + { + if (m_test_drop_download_height == 0) + return true; + + if (get_blockchain_storage().get_current_blockchain_height() <= m_test_drop_download_height) + return true; + + return false; + } + //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block) { tvc = boost::value_initialized<tx_verification_context>(); @@ -290,9 +421,9 @@ namespace cryptonote return false; } - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - LOG_PRINT_RED_L1("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + LOG_PRINT_RED_L1("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -527,6 +658,11 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const + { + return m_mempool.get_transactions_and_spent_keys_info(tx_infos, key_image_infos); + } + //----------------------------------------------------------------------------------------------- bool core::get_short_chain_history(std::list<crypto::hash>& ids) { return m_blockchain_storage.get_short_chain_history(ids); @@ -577,7 +713,11 @@ namespace cryptonote m_starter_message_showed = true; } +#if BLOCKCHAIN_DB == DB_LMDB + m_store_blockchain_interval.do_call(boost::bind(&Blockchain::store_blockchain, &m_blockchain_storage)); +#else m_store_blockchain_interval.do_call(boost::bind(&blockchain_storage::store_blockchain, &m_blockchain_storage)); +#endif m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -595,4 +735,6 @@ namespace cryptonote { raise(SIGTERM); } + + std::atomic<bool> core::m_fast_exit(false); } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 10522451c..ff187b4ce 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -39,7 +39,11 @@ #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "tx_pool.h" +#if BLOCKCHAIN_DB == DB_LMDB +#include "blockchain.h" +#else #include "blockchain_storage.h" +#endif #include "miner.h" #include "connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" @@ -72,9 +76,15 @@ namespace cryptonote miner& get_miner(){return m_miner;} static void init_options(boost::program_options::options_description& desc); - bool init(const boost::program_options::variables_map& vm, bool testnet); + bool init(const boost::program_options::variables_map& vm); bool set_genesis_block(const block& b); bool deinit(); + static void set_fast_exit(); + static bool get_fast_exit(); + void test_drop_download(); + void test_drop_download_height(uint64_t height); + bool get_test_drop_download(); + bool get_test_drop_download_height(); uint64_t get_current_blockchain_height(); bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id); bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs); @@ -98,6 +108,7 @@ namespace cryptonote void set_enforce_dns_checkpoints(bool enforce_dns); bool get_pool_transactions(std::list<transaction>& txs); + bool get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const; size_t get_pool_transactions_count(); size_t get_blockchain_total_transactions(); //bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys); @@ -112,7 +123,11 @@ namespace cryptonote bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); void pause_mine(); void resume_mine(); +#if BLOCKCHAIN_DB == DB_LMDB + Blockchain& get_blockchain_storage(){return m_blockchain_storage;} +#else blockchain_storage& get_blockchain_storage(){return m_blockchain_storage;} +#endif //debug functions void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); @@ -125,6 +140,8 @@ namespace cryptonote bool update_checkpoints(); + void stop(); + private: bool add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); bool add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block); @@ -142,14 +159,20 @@ namespace cryptonote bool check_tx_ring_signature(const txin_to_key& tx, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig); bool is_tx_spendtime_unlocked(uint64_t unlock_time); bool update_miner_block_template(); - bool handle_command_line(const boost::program_options::variables_map& vm, bool testnet); + bool handle_command_line(const boost::program_options::variables_map& vm); bool on_update_blocktemplate_interval(); bool check_tx_inputs_keyimages_diff(const transaction& tx); void graceful_exit(); - + static std::atomic<bool> m_fast_exit; + bool m_test_drop_download = true; + uint64_t m_test_drop_download_height = 0; tx_memory_pool m_mempool; +#if BLOCKCHAIN_DB == DB_LMDB + Blockchain m_blockchain_storage; +#else blockchain_storage m_blockchain_storage; +#endif i_cryptonote_protocol* m_pprotocol; epee::critical_section m_incoming_tx_lock; //m_miner and m_miner_addres are probably temporary here @@ -163,9 +186,12 @@ namespace cryptonote uint64_t m_target_blockchain_height; + bool m_testnet; std::string m_checkpoints_path; time_t m_last_dns_checkpoints_update; time_t m_last_json_checkpoints_update; + + std::atomic_flag m_checkpoints_updating; }; } diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp index 8c7b2fbaa..1b21894a0 100644 --- a/src/cryptonote_core/cryptonote_format_utils.cpp +++ b/src/cryptonote_core/cryptonote_format_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_format_utils.h b/src/cryptonote_core/cryptonote_format_utils.h index 9fa7f0545..fd785aafd 100644 --- a/src/cryptonote_core/cryptonote_format_utils.h +++ b/src/cryptonote_core/cryptonote_format_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_stat_info.h b/src/cryptonote_core/cryptonote_stat_info.h index 5d0d1769d..20bbe382e 100644 --- a/src/cryptonote_core/cryptonote_stat_info.h +++ b/src/cryptonote_core/cryptonote_stat_info.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/difficulty.cpp b/src/cryptonote_core/difficulty.cpp index d4d733e69..6486d8124 100644 --- a/src/cryptonote_core/difficulty.cpp +++ b/src/cryptonote_core/difficulty.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -56,10 +56,40 @@ namespace cryptonote { #else static inline void mul(uint64_t a, uint64_t b, uint64_t &low, uint64_t &high) { - typedef unsigned __int128 uint128_t; - uint128_t res = (uint128_t) a * (uint128_t) b; - low = (uint64_t) res; - high = (uint64_t) (res >> 64); + // __int128 isn't part of the standard, so the previous function wasn't portable. mul128() in Windows is fine, + // but this portable function should be used elsewhere. Credit for this function goes to latexi95. + + uint64_t aLow = a & 0xFFFFFFFF; + uint64_t aHigh = a >> 32; + uint64_t bLow = b & 0xFFFFFFFF; + uint64_t bHigh = b >> 32; + + uint64_t res = aLow * bLow; + uint64_t lowRes1 = res & 0xFFFFFFFF; + uint64_t carry = res >> 32; + + res = aHigh * bLow + carry; + uint64_t highResHigh1 = res >> 32; + uint64_t highResLow1 = res & 0xFFFFFFFF; + + res = aLow * bHigh; + uint64_t lowRes2 = res & 0xFFFFFFFF; + carry = res >> 32; + + res = aHigh * bHigh + carry; + uint64_t highResHigh2 = res >> 32; + uint64_t highResLow2 = res & 0xFFFFFFFF; + + //Addition + + uint64_t r = highResLow1 + lowRes2; + carry = r >> 32; + low = (r << 32) | lowRes1; + r = highResHigh1 + highResLow2 + carry; + uint64_t d3 = r & 0xFFFFFFFF; + carry = r >> 32; + r = highResHigh2 + carry; + high = d3 | (r << 32); } #endif diff --git a/src/cryptonote_core/difficulty.h b/src/cryptonote_core/difficulty.h index 99c9228f0..42d2df500 100644 --- a/src/cryptonote_core/difficulty.h +++ b/src/cryptonote_core/difficulty.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/miner.cpp b/src/cryptonote_core/miner.cpp index b2adfe39c..b99113d9e 100644 --- a/src/cryptonote_core/miner.cpp +++ b/src/cryptonote_core/miner.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/miner.h b/src/cryptonote_core/miner.h index 44844428c..7ada8171f 100644 --- a/src/cryptonote_core/miner.h +++ b/src/cryptonote_core/miner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/tx_extra.h b/src/cryptonote_core/tx_extra.h index af91deca5..ccfe4d1d9 100644 --- a/src/cryptonote_core/tx_extra.h +++ b/src/cryptonote_core/tx_extra.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 96c6ebed9..5543ac64f 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -37,7 +37,11 @@ #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" +#if BLOCKCHAIN_DB == DB_LMDB +#include "blockchain.h" +#else #include "blockchain_storage.h" +#endif #include "common/boost_serialization_helper.h" #include "common/int-util.h" #include "misc_language.h" @@ -52,12 +56,19 @@ namespace cryptonote { size_t const TRANSACTION_SIZE_LIMIT = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); } - //--------------------------------------------------------------------------------- +#if BLOCKCHAIN_DB == DB_LMDB + //--------------------------------------------------------------------------------- + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs) + { + + } +#else tx_memory_pool::tx_memory_pool(blockchain_storage& bchs): m_blockchain(bchs) { } +#endif //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(const transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block) { @@ -246,6 +257,7 @@ namespace cryptonote (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && it->second.kept_by_block) ) { LOG_PRINT_L1("Tx " << it->first << " removed from tx pool due to outdated, age: " << tx_age ); + remove_transaction_keyimages(it->second.tx); m_transactions.erase(it++); }else ++it; @@ -265,6 +277,40 @@ namespace cryptonote BOOST_FOREACH(const auto& tx_vt, m_transactions) txs.push_back(tx_vt.second.tx); } + //------------------------------------------------------------------ + bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + for (const auto& tx_vt : m_transactions) + { + tx_info txi; + const tx_details& txd = tx_vt.second; + txi.id_hash = epee::string_tools::pod_to_hex(tx_vt.first); + txi.tx_json = obj_to_json_str(*const_cast<transaction*>(&txd.tx)); + txi.blob_size = txd.blob_size; + txi.fee = txd.fee; + txi.kept_by_block = txd.kept_by_block; + txi.max_used_block_height = txd.max_used_block_height; + txi.max_used_block_id_hash = epee::string_tools::pod_to_hex(txd.max_used_block_id); + txi.last_failed_height = txd.last_failed_height; + txi.last_failed_id_hash = epee::string_tools::pod_to_hex(txd.last_failed_id); + txi.receive_time = txd.receive_time; + tx_infos.push_back(txi); + } + + for (const key_images_container::value_type& kee : m_spent_key_images) { + const crypto::key_image& k_image = kee.first; + const std::unordered_set<crypto::hash>& kei_image_set = kee.second; + spent_key_image_info ki; + ki.id_hash = epee::string_tools::pod_to_hex(k_image); + for (const crypto::hash& tx_id_hash : kei_image_set) + { + ki.txs_hashes.push_back(epee::string_tools::pod_to_hex(tx_id_hash)); + } + key_image_infos.push_back(ki); + } + return true; + } //--------------------------------------------------------------------------------- bool tx_memory_pool::get_transaction(const crypto::hash& id, transaction& tx) const { diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 7617765cc..77f2e3483 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -43,11 +43,15 @@ #include "cryptonote_basic_impl.h" #include "verification_context.h" #include "crypto/hash.h" - +#include "rpc/core_rpc_server_commands_defs.h" namespace cryptonote { +#if BLOCKCHAIN_DB == DB_LMDB + class Blockchain; +#else class blockchain_storage; +#endif /************************************************************************/ /* */ /************************************************************************/ @@ -55,7 +59,11 @@ namespace cryptonote class tx_memory_pool: boost::noncopyable { public: +#if BLOCKCHAIN_DB == DB_LMDB + tx_memory_pool(Blockchain& bchs); +#else tx_memory_pool(blockchain_storage& bchs); +#endif bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); bool add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block); //gets tx and remove it from pool @@ -74,6 +82,7 @@ namespace cryptonote bool deinit(); bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee); void get_transactions(std::list<transaction>& txs) const; + bool get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const; bool get_transaction(const crypto::hash& h, transaction& tx) const; size_t get_transactions_count() const; std::string print_pool(bool short_format) const; @@ -127,7 +136,11 @@ namespace cryptonote //transactions_container m_alternative_transactions; std::string m_config_folder; +#if BLOCKCHAIN_DB == DB_LMDB + Blockchain& m_blockchain; +#else blockchain_storage& m_blockchain; +#endif /************************************************************************/ /* */ /************************************************************************/ @@ -170,9 +183,12 @@ namespace cryptonote uint64_t operator()(const txin_to_scripthash& tx) const {return 0;} }; +#if BLOCKCHAIN_DB == DB_LMDB +#else #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) friend class blockchain_storage; #endif +#endif }; } diff --git a/src/cryptonote_core/verification_context.h b/src/cryptonote_core/verification_context.h index 3be45907b..c467d35f3 100644 --- a/src/cryptonote_core/verification_context.h +++ b/src/cryptonote_core/verification_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt new file mode 100644 index 000000000..2ea5662a1 --- /dev/null +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2014, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +cmake_minimum_required (VERSION 2.6) +project (bitmonero CXX) + +file(GLOB CRYPTONOTE_PROTOCOL *) +source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL}) + +#add_library(p2p ${P2P}) + +#bitmonero_private_headers(p2p ${CRYPTONOTE_PROTOCOL}) +bitmonero_add_library(cryptonote_protocol ${CRYPTONOTE_PROTOCOL}) +#target_link_libraries(p2p) +# LINK_PRIVATE +# ${Boost_CHRONO_LIBRARY} +# ${Boost_REGEX_LIBRARY} +# ${Boost_SYSTEM_LIBRARY} +# ${Boost_THREAD_LIBRARY} +# ${EXTRA_LIBRARIES}) +add_dependencies(cryptonote_protocol + version) diff --git a/src/cryptonote_protocol/blobdatatype.h b/src/cryptonote_protocol/blobdatatype.h index 7cde6bdb3..063f08ef0 100644 --- a/src/cryptonote_protocol/blobdatatype.h +++ b/src/cryptonote_protocol/blobdatatype.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index 66fa43dd2..7e019b533 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -46,6 +46,8 @@ namespace cryptonote struct connection_info { bool incoming; + bool localhost; + bool local_ip; std::string ip; std::string port; @@ -62,8 +64,16 @@ namespace cryptonote uint64_t live_time; + uint64_t avg_download; + uint64_t current_download; + + uint64_t avg_upload; + uint64_t current_upload; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(incoming) + KV_SERIALIZE(localhost) + KV_SERIALIZE(local_ip) KV_SERIALIZE(ip) KV_SERIALIZE(port) KV_SERIALIZE(peer_id) @@ -73,6 +83,10 @@ namespace cryptonote KV_SERIALIZE(send_idle_time) KV_SERIALIZE(state) KV_SERIALIZE(live_time) + KV_SERIALIZE(avg_download) + KV_SERIALIZE(current_download) + KV_SERIALIZE(avg_upload) + KV_SERIALIZE(current_upload) END_KV_SERIALIZE_MAP() }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp new file mode 100644 index 000000000..614ee8fab --- /dev/null +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -0,0 +1,176 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> + +#include "../../src/cryptonote_protocol/cryptonote_protocol_handler.h" +#include "../../src/p2p/network_throttle.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + +#include "../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +class cryptonote_protocol_handler_base_pimpl { // placeholer if needed + public: + +}; + +} // namespace + +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa + const double size_min = 500; // XXX 500 + //const int history_len = 20; // how many blocks to average over + + double avg=0; + try { + avg = get_avg_block_size(/*history_len*/); + } catch (...) { } + avg = std::max( size_min , avg); + return avg; +} + +cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { +} + +cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { +} + +void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list<crypto::hash>& ids) { + using namespace epee::net_utils; + LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + LOG_PRINT_RED("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)" , LOG_LEVEL_0); + _note_c("net/req2", "### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + // TODO +} + +void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_dbg1(""); + using namespace epee::net_utils; + double delay=0; // will be calculated + _dbg1("Packet size: " << packet_size); + do + { // rate limiting + //XXX + /*if (::cryptonote::core::get_is_stopping()) { + _dbg1("We are stopping - so abort sleep"); + return; + }*/ + /*if (m_was_shutdown) { + _dbg2_c("net/netuse/sleep","m_was_shutdown - so abort sleep"); + return; + }*/ + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + + delay *= 0.50; + //delay = 0; // XXX + if (delay > 0) { + //delay += rand2*0.1; + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // XXX debug sleep + _dbg1_c("net/sleep/", "sleep in sleep_before_packet"); + _dbg2("Sleep for " << ms); + boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps + } + } while(delay > 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global + //epee::critical_region_t<decltype(m_throttle_global_lock)> guard(m_throttle_global_lock); // *** critical *** + //m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global + } +} + +} // namespace + + diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 0e1a6f1b3..571c36dc1 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -1,4 +1,8 @@ -// Copyright (c) 2014, The Monero Project +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -41,15 +45,36 @@ #include "cryptonote_core/connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" +// #include <netinet/in.h> +#include <boost/circular_buffer.hpp> PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) +#define LOCALHOST_INT 2130706433 + namespace cryptonote { + class cryptonote_protocol_handler_base_pimpl; + class cryptonote_protocol_handler_base { + private: + std::unique_ptr<cryptonote_protocol_handler_base_pimpl> mI; + + public: + cryptonote_protocol_handler_base(); + virtual ~cryptonote_protocol_handler_base(); + void handler_request_blocks_history(std::list<crypto::hash>& ids); // before asking for list of objects, we can change the list still + void handler_response_blocks_now(size_t packet_size); + + virtual double get_avg_block_size() = 0; + virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download + + virtual std::ofstream& get_logreq() const =0; + }; + template<class t_core> - class t_cryptonote_protocol_handler: public i_cryptonote_protocol + class t_cryptonote_protocol_handler: public i_cryptonote_protocol, cryptonote_protocol_handler_base { public: typedef cryptonote_connection_context connection_context; @@ -106,6 +131,12 @@ namespace cryptonote nodetool::i_p2p_endpoint<connection_context>* m_p2p; std::atomic<uint32_t> m_syncronized_connections_count; std::atomic<bool> m_synchronized; + bool m_one_request = true; + + // static std::ofstream m_logreq; + std::mutex m_buffer_mutex; + double get_avg_block_size(); + boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10); template<class t_parametr> bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context) @@ -113,6 +144,7 @@ namespace cryptonote LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parametr).name() << " -->"); std::string blob; epee::serialization::store_t_to_binary(arg, blob); + //handler_response_blocks_now(blob.size()); // XXX return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context); } @@ -124,8 +156,11 @@ namespace cryptonote epee::serialization::store_t_to_binary(arg, arg_buff); return m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context); } + + virtual std::ofstream& get_logreq() const ; }; -} + +} // namespace #include "cryptonote_protocol_handler.inl" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 19ad74368..1cf66521f 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1,4 +1,8 @@ -// Copyright (c) 2014, The Monero Project +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -28,14 +32,28 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +// (may contain code and/or modifications by other developers) +// developer rfree: this code is caller of our new network code, and is modded; e.g. for rate limiting + #include <boost/interprocess/detail/atomic.hpp> #include <list> #include "cryptonote_core/cryptonote_format_utils.h" #include "profile_tools.h" +#include "../../contrib/otshell_utils/utils.hpp" +#include "../../src/p2p/network_throttle-detail.hpp" +#include "../../src/p2p/data_logger.hpp" +using namespace nOT::nUtils; + namespace cryptonote { + +// static +// template<class t_core> std::ofstream t_cryptonote_protocol_handler<t_core>::m_logreq("logreq.txt"); // static + + + //----------------------------------------------------------------------------------------------------------------------- template<class t_core> t_cryptonote_protocol_handler<t_core>::t_cryptonote_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint<connection_context>* p_net_layout):m_core(rcore), @@ -99,24 +117,66 @@ namespace cryptonote void t_cryptonote_protocol_handler<t_core>::log_connections() { std::stringstream ss; + ss.precision(1); + + double down_sum = 0.0; + double down_curr_sum = 0.0; + double up_sum = 0.0; + double up_curr_sum = 0.0; - ss << std::setw(25) << std::left << "Remote Host" + ss << std::setw(30) << std::left << "Remote Host" << std::setw(20) << "Peer id" - << std::setw(25) << "Recv/Sent (inactive,sec)" + << std::setw(30) << "Recv/Sent (inactive,sec)" << std::setw(25) << "State" - << std::setw(20) << "Livetime(seconds)" << ENDL; - + << std::setw(20) << "Livetime(sec)" + << std::setw(12) << "Down (kB/s)" + << std::setw(14) << "Down(now)" + << std::setw(10) << "Up (kB/s)" + << std::setw(13) << "Up(now)" + << ENDL; + + uint32_t ip; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id) { - ss << std::setw(25) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + - epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) + bool local_ip = false; + ip = ntohl(cntxt.m_remote_ip); + // TODO: local ip in calss A, B + if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x + local_ip = true; + auto connection_time = time(NULL) - cntxt.m_started; + ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + + epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) << std::setw(20) << std::hex << peer_id - << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" + << std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) - << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) << ENDL; + << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) + << std::setw(12) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_recv_cnt / connection_time / 1024) + << std::setw(14) << std::fixed << cntxt.m_current_speed_down / 1024 + << std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024) + << std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024 + << (local_ip ? "[LAN]" : "") + << std::left << (ip == LOCALHOST_INT ? "[LOCALHOST]" : "") // 127.0.0.1 + << ENDL; + + if (connection_time > 1) + { + down_sum += (cntxt.m_recv_cnt / connection_time / 1024); + up_sum += (cntxt.m_send_cnt / connection_time / 1024); + } + + down_curr_sum += (cntxt.m_current_speed_down / 1024); + up_curr_sum += (cntxt.m_current_speed_up / 1024); + return true; }); - LOG_PRINT_L0("Connections: " << ENDL << ss.str()); + ss << ENDL + << std::setw(125) << " " + << std::setw(12) << down_sum + << std::setw(14) << down_curr_sum + << std::setw(10) << up_sum + << std::setw(13) << up_curr_sum + << ENDL; + LOG_PRINT_L0("Connections: " << ENDL << ss.str()); } //------------------------------------------------------------------------------------------------------------------------ // Returns a list of connection_info objects describing each open p2p connection @@ -150,6 +210,42 @@ namespace cryptonote cnx.live_time = timestamp - cntxt.m_started; + uint32_t ip; + ip = ntohl(cntxt.m_remote_ip); + if (ip == LOCALHOST_INT) + { + cnx.localhost = true; + } + else + { + cnx.localhost = false; + } + + if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x + { + cnx.local_ip = true; + } + else + { + cnx.local_ip = false; + } + + auto connection_time = time(NULL) - cntxt.m_started; + if (connection_time == 0) + { + cnx.avg_download = 0; + cnx.avg_upload = 0; + } + + else + { + cnx.avg_download = cntxt.m_recv_cnt / connection_time / 1024; + cnx.avg_upload = cntxt.m_send_cnt / connection_time / 1024; + } + + cnx.current_download = cntxt.m_current_speed_down / 1024; + cnx.current_upload = cntxt.m_current_speed_up / 1024; + connections.push_back(cnx); return true; @@ -234,11 +330,11 @@ namespace cryptonote block_verification_context bvc = boost::value_initialized<block_verification_context>(); m_core.pause_mine(); - m_core.handle_incoming_block(arg.b.block, bvc); + m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block m_core.resume_mine(); if(bvc.m_verifivation_failed) { - LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); + LOG_PRINT_CCONTEXT_L0("Block verification failed, dropping connection"); m_p2p->drop_connection(context); return 1; } @@ -304,13 +400,70 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify<NOTIFY_RESPONSE_GET_OBJECTS>(rsp, context); + //handler_response_blocks_now(sizeof(rsp)); // XXX + //handler_response_blocks_now(200); return 1; } //------------------------------------------------------------------------------------------------------------------------ + + + template<class t_core> + double t_cryptonote_protocol_handler<t_core>::get_avg_block_size() { + // return m_core.get_blockchain_storage().get_avg_block_size(count); // this does not count too well the actuall network-size of data we need to download + + CRITICAL_REGION_LOCAL(m_buffer_mutex); + double avg = 0; + if (m_avg_buffer.size() == 0) { + _warn("m_avg_buffer.size() == 0"); + return 500; + } + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + for (auto element : m_avg_buffer) avg += element; + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + return avg / m_avg_buffer.size(); + } + + template<class t_core> int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { LOG_PRINT_CCONTEXT_L2("NOTIFY_RESPONSE_GET_OBJECTS"); + + // calculate size of request - mainly for logging/debug + size_t size = 0; + for (auto element : arg.txs) size += element.size(); + + for (auto element : arg.blocks) { + size += element.block.size(); + for (auto tx : element.txs) + size += tx.size(); + } + + for (auto element : arg.missed_ids) + size += sizeof(element.data); + + size += sizeof(arg.current_blockchain_height); + { + CRITICAL_REGION_LOCAL(m_buffer_mutex); + m_avg_buffer.push_back(size); + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + m_avg_buffer.push_back(666); // a test value + m_avg_buffer.erase_end(1); + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + } + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + if(context.m_last_response_height > arg.current_blockchain_height) { LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height @@ -373,53 +526,67 @@ namespace cryptonote return 1; } + { m_core.pause_mine(); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( boost::bind(&t_core::resume_mine, &m_core)); - BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) - { - //process transactions - TIME_MEASURE_START(transactions_process_time); - BOOST_FOREACH(auto& tx_blob, block_entry.txs) - { - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(tx_blob, tvc, true); - if(tvc.m_verifivation_failed) - { - LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - } - TIME_MEASURE_FINISH(transactions_process_time); - - //process block - TIME_MEASURE_START(block_process_time); - block_verification_context bvc = boost::value_initialized<block_verification_context>(); - - m_core.handle_incoming_block(block_entry.block, bvc, false); - - if(bvc.m_verifivation_failed) - { - LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - if(bvc.m_marked_as_orphaned) - { - LOG_PRINT_CCONTEXT_L1("Block received at sync phase was marked as orphaned, dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - - TIME_MEASURE_FINISH(block_process_time); - LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); - } + LOG_PRINT_CCONTEXT_YELLOW( "Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() , LOG_LEVEL_0); + + if (m_core.get_test_drop_download() && m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing + + + BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) + { + // process transactions + TIME_MEASURE_START(transactions_process_time); + BOOST_FOREACH(auto& tx_blob, block_entry.txs) + { + tx_verification_context tvc = AUTO_VAL_INIT(tvc); + m_core.handle_incoming_tx(tx_blob, tvc, true); + if(tvc.m_verifivation_failed) + { + LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " + << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + } + TIME_MEASURE_FINISH(transactions_process_time); + + // process block + + TIME_MEASURE_START(block_process_time); + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + + m_core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block + + if(bvc.m_verifivation_failed) + { + LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + if(bvc.m_marked_as_orphaned) + { + LOG_PRINT_CCONTEXT_L1("Block received at sync phase was marked as orphaned, dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + + TIME_MEASURE_FINISH(block_process_time); + LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); + + epee::net_utils::data_logger::get_instance().add_data("calc_time", block_process_time + transactions_process_time); + epee::net_utils::data_logger::get_instance().add_data("block_processing", 1); + + } // each download block + + } // if not DISCARD BLOCK + + } - request_missing_objects(context, true); return 1; } @@ -448,6 +615,15 @@ namespace cryptonote template<class t_core> bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks) { + //if (!m_one_request == false) + //return true; + m_one_request = false; + // save request size to log (dr monero) + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + if(context.m_needed_objects.size()) { //we know objects that we need, request this objects @@ -455,6 +631,8 @@ namespace cryptonote size_t count = 0; auto it = context.m_needed_objects.begin(); + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + _note_c("net/req-calc" , "Setting count_limit: " << count_limit); while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT) { if( !(check_having_blocks && m_core.have_block(*it))) @@ -465,14 +643,24 @@ namespace cryptonote } context.m_needed_objects.erase(it++); } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()); + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + << "requested blocks count=" << count << " / " << count_limit); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); }else if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + + //std::string blob; // for calculate size of request + //epee::serialization::store_t_to_binary(r, blob); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + LOG_PRINT_CCONTEXT_L0("r = " << 200); + + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); }else { @@ -575,4 +763,18 @@ namespace cryptonote { return relay_post_notify<NOTIFY_NEW_TRANSACTIONS>(arg, exclude_context); } -} + + /// @deprecated + template<class t_core> std::ofstream& t_cryptonote_protocol_handler<t_core>::get_logreq() const { + static std::ofstream * logreq=NULL; + if (!logreq) { + LOG_PRINT_RED("LOG OPENED",LOG_LEVEL_0); + logreq = new std::ofstream("logreq.txt"); // leak mem (singleton) + *logreq << "Opened log" << std::endl; + } + LOG_PRINT_YELLOW("LOG USED",LOG_LEVEL_0); + (*logreq) << "log used" << std::endl; + return *logreq; + } + +} // namespace diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 4532645ce..9ccf6aa56 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index adcc61c43..bf25bbca9 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -27,12 +27,27 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set(daemon_sources - daemon.cpp) + command_parser_executor.cpp + command_server.cpp + daemon.cpp + executor.cpp + main.cpp + rpc_command_executor.cpp +) set(daemon_headers) set(daemon_private_headers + command_parser_executor.h + command_server.h + core.h + daemon.h daemon_commands_handler.h + executor.h + p2p.h + protocol.h + rpc.h + rpc_command_executor.h # cryptonote_protocol ../cryptonote_protocol/blobdatatype.h @@ -59,9 +74,14 @@ bitmonero_add_executable(daemon target_link_libraries(daemon LINK_PRIVATE rpc + blockchain_db cryptonote_core crypto common + otshell_utils + p2p + cryptonote_protocol + daemonizer ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h new file mode 100644 index 000000000..2bd918478 --- /dev/null +++ b/src/daemon/command_line_args.h @@ -0,0 +1,81 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef DAEMON_COMMAND_LINE_ARGS_H +#define DAEMON_COMMAND_LINE_ARGS_H + +#include "common/command_line.h" +#include "cryptonote_config.h" +#include <boost/program_options.hpp> + +namespace daemon_args +{ + std::string const WINDOWS_SERVICE_NAME = "Monero Daemon"; + + const command_line::arg_descriptor<std::string> arg_config_file = { + "config-file" + , "Specify configuration file" + , std::string(CRYPTONOTE_NAME ".conf") + }; + const command_line::arg_descriptor<std::string> arg_log_file = { + "log-file" + , "Specify log file" + , "" + }; + const command_line::arg_descriptor<int> arg_log_level = { + "log-level" + , "" + , LOG_LEVEL_0 + }; + const command_line::arg_descriptor<std::vector<std::string>> arg_command = { + "daemon_command" + , "Hidden" + }; + const command_line::arg_descriptor<bool> arg_os_version = { + "os-version" + , "OS for which this executable was compiled" + }; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet. The wallet must be launched with --testnet flag." + , false + }; + const command_line::arg_descriptor<bool> arg_dns_checkpoints = { + "enforce-dns-checkpointing" + , "checkpoints from DNS server will be enforced" + , false + }; + const command_line::arg_descriptor<std::string> arg_db_type = { + "db-type" + , "Specify database type" + , "lmdb" + }; + +} // namespace daemon_args + +#endif // DAEMON_COMMAND_LINE_ARGS_H diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp new file mode 100644 index 000000000..fd7b40be2 --- /dev/null +++ b/src/daemon/command_parser_executor.cpp @@ -0,0 +1,336 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_core/cryptonote_basic_impl.h" +#include "daemon/command_parser_executor.h" + +namespace daemonize { + +t_command_parser_executor::t_command_parser_executor( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server + ) + : m_executor(ip, port, is_rpc, rpc_server) +{} + +bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_peer_list(); +} + +bool t_command_parser_executor::save_blockchain(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.save_blockchain(); +} + +bool t_command_parser_executor::show_hash_rate(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.show_hash_rate(); +} + +bool t_command_parser_executor::hide_hash_rate(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.hide_hash_rate(); +} + +bool t_command_parser_executor::show_difficulty(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.show_difficulty(); +} + +bool t_command_parser_executor::print_connections(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_connections(); +} + +bool t_command_parser_executor::print_blockchain_info(const std::vector<std::string>& args) +{ + if(!args.size()) + { + std::cout << "need block index parameter" << std::endl; + return false; + } + uint64_t start_index = 0; + uint64_t end_index = 0; + if(!epee::string_tools::get_xtype_from_string(start_index, args[0])) + { + std::cout << "wrong starter block index parameter" << std::endl; + return false; + } + if(args.size() >1 && !epee::string_tools::get_xtype_from_string(end_index, args[1])) + { + std::cout << "wrong end block index parameter" << std::endl; + return false; + } + + return m_executor.print_blockchain_info(start_index, end_index); +} + +bool t_command_parser_executor::set_log_level(const std::vector<std::string>& args) +{ + if(args.size() != 1) + { + std::cout << "use: set_log <log_level_number_0-4>" << std::endl; + return true; + } + + uint16_t l = 0; + if(!epee::string_tools::get_xtype_from_string(l, args[0])) + { + std::cout << "wrong number format, use: set_log <log_level_number_0-4>" << std::endl; + return true; + } + + if(LOG_LEVEL_4 < l) + { + std::cout << "wrong number range, use: set_log <log_level_number_0-4>" << std::endl; + return true; + } + + return m_executor.set_log_level(l); +} + +bool t_command_parser_executor::print_height(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_height(); +} + +bool t_command_parser_executor::print_block(const std::vector<std::string>& args) +{ + if (args.empty()) + { + std::cout << "expected: print_block (<block_hash> | <block_height>)" << std::endl; + return false; + } + + const std::string& arg = args.front(); + try + { + uint64_t height = boost::lexical_cast<uint64_t>(arg); + return m_executor.print_block_by_height(height); + } + catch (boost::bad_lexical_cast&) + { + crypto::hash block_hash; + if (parse_hash256(arg, block_hash)) + { + return m_executor.print_block_by_hash(block_hash); + } + } + + return false; +} + +bool t_command_parser_executor::print_transaction(const std::vector<std::string>& args) +{ + if (args.empty()) + { + std::cout << "expected: print_tx <transaction hash>" << std::endl; + return true; + } + + const std::string& str_hash = args.front(); + crypto::hash tx_hash; + if (parse_hash256(str_hash, tx_hash)) + { + m_executor.print_transaction(tx_hash); + } + + return true; +} + +bool t_command_parser_executor::print_transaction_pool_long(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_transaction_pool_long(); +} + +bool t_command_parser_executor::print_transaction_pool_short(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_transaction_pool_short(); +} + +bool t_command_parser_executor::start_mining(const std::vector<std::string>& args) +{ + if(!args.size()) + { + std::cout << "Please specify a wallet address to mine for: start_mining <addr> [threads=1]" << std::endl; + return true; + } + + cryptonote::account_public_address adr; + if(!cryptonote::get_account_address_from_str(adr, false, args.front())) + { + if(!cryptonote::get_account_address_from_str(adr, true, args.front())) + { + std::cout << "target account address has wrong format" << std::endl; + return true; + } + std::cout << "Mining to a testnet address, make sure this is intentional!" << std::endl; + } + uint64_t threads_count = 1; + if(args.size() > 2) + { + return false; + } + else if(args.size() == 2) + { + bool ok = epee::string_tools::get_xtype_from_string(threads_count, args[1]); + threads_count = (ok && 0 < threads_count) ? threads_count : 1; + } + + m_executor.start_mining(adr, threads_count); + + return true; +} + +bool t_command_parser_executor::stop_mining(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.stop_mining(); +} + +bool t_command_parser_executor::stop_daemon(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.stop_daemon(); +} + +bool t_command_parser_executor::print_status(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_status(); +} + +bool t_command_parser_executor::set_limit(const std::vector<std::string>& args) +{ + if(args.size()!=1) return false; + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + if (limit==-1) limit=128; + limit *= 1024; + + return m_executor.set_limit(limit); +} + +bool t_command_parser_executor::set_limit_up(const std::vector<std::string>& args) +{ + if(args.size()!=1) return false; + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + if (limit==-1) limit=128; + limit *= 1024; + + return m_executor.set_limit_up(limit); +} + +bool t_command_parser_executor::set_limit_down(const std::vector<std::string>& args) +{ + if(args.size()!=1) return false; + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + if (limit==-1) limit=128; + limit *= 1024; + + return m_executor.set_limit_down(limit); +} + +bool t_command_parser_executor::fast_exit(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + return m_executor.fast_exit(); +} + +bool t_command_parser_executor::out_peers(const std::vector<std::string>& args) +{ + if (args.empty()) return false; + + unsigned int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + _erro("stoi exception"); + return false; + } + + return m_executor.out_peers(limit); +} + +bool t_command_parser_executor::start_save_graph(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + return m_executor.start_save_graph(); +} + +bool t_command_parser_executor::stop_save_graph(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + return m_executor.stop_save_graph(); +} + + +} // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h new file mode 100644 index 000000000..27ffabc1c --- /dev/null +++ b/src/daemon/command_parser_executor.h @@ -0,0 +1,105 @@ +/** +@file +@details + +@image html images/other/runtime-commands.png + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "daemon/rpc_command_executor.h" +#include "rpc/core_rpc_server.h" + +namespace daemonize { + +class t_command_parser_executor final +{ +private: + t_rpc_command_executor m_executor; +public: + t_command_parser_executor( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server = NULL + ); + + bool print_peer_list(const std::vector<std::string>& args); + + bool save_blockchain(const std::vector<std::string>& args); + + bool show_hash_rate(const std::vector<std::string>& args); + + bool hide_hash_rate(const std::vector<std::string>& args); + + bool show_difficulty(const std::vector<std::string>& args); + + bool print_connections(const std::vector<std::string>& args); + + bool print_blockchain_info(const std::vector<std::string>& args); + + bool set_log_level(const std::vector<std::string>& args); + + bool print_height(const std::vector<std::string>& args); + + bool print_block(const std::vector<std::string>& args); + + bool print_transaction(const std::vector<std::string>& args); + + bool print_transaction_pool_long(const std::vector<std::string>& args); + + bool print_transaction_pool_short(const std::vector<std::string>& args); + + bool start_mining(const std::vector<std::string>& args); + + bool stop_mining(const std::vector<std::string>& args); + + bool stop_daemon(const std::vector<std::string>& args); + + bool print_status(const std::vector<std::string>& args); + + bool set_limit(const std::vector<std::string>& args); + + bool set_limit_up(const std::vector<std::string>& args); + + bool set_limit_down(const std::vector<std::string>& args); + + bool fast_exit(const std::vector<std::string>& args); + + bool out_peers(const std::vector<std::string>& args); + + bool start_save_graph(const std::vector<std::string>& args); + + bool stop_save_graph(const std::vector<std::string>& args); +}; + +} // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp new file mode 100644 index 000000000..f0f4cd676 --- /dev/null +++ b/src/daemon/command_server.cpp @@ -0,0 +1,233 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_config.h" +#include "version.h" +#include "daemon/command_server.h" + +namespace daemonize { + +namespace p = std::placeholders; + +t_command_server::t_command_server( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server + ) + : m_parser(ip, port, is_rpc, rpc_server) + , m_command_lookup() + , m_is_rpc(is_rpc) +{ + m_command_lookup.set_handler( + "q" + , [] (const std::vector<std::string>& args) {return true;} + , "ignored" + ); + m_command_lookup.set_handler( + "help" + , std::bind(&t_command_server::help, this, p::_1) + , "Show this help" + ); + m_command_lookup.set_handler( + "print_height" + , std::bind(&t_command_parser_executor::print_height, &m_parser, p::_1) + , "Print local blockchain height" + ); + m_command_lookup.set_handler( + "print_pl" + , std::bind(&t_command_parser_executor::print_peer_list, &m_parser, p::_1) + , "Print peer list" + ); + m_command_lookup.set_handler( + "print_cn" + , std::bind(&t_command_parser_executor::print_connections, &m_parser, p::_1) + , "Print connections" + ); + m_command_lookup.set_handler( + "print_bc" + , std::bind(&t_command_parser_executor::print_blockchain_info, &m_parser, p::_1) + , "Print blockchain info in a given blocks range, print_bc <begin_height> [<end_height>]" + ); + m_command_lookup.set_handler( + "print_block" + , std::bind(&t_command_parser_executor::print_block, &m_parser, p::_1) + , "Print block, print_block <block_hash> | <block_height>" + ); + m_command_lookup.set_handler( + "print_tx" + , std::bind(&t_command_parser_executor::print_transaction, &m_parser, p::_1) + , "Print transaction, print_tx <transaction_hash>" + ); + m_command_lookup.set_handler( + "start_mining" + , std::bind(&t_command_parser_executor::start_mining, &m_parser, p::_1) + , "Start mining for specified address, start_mining <addr> [threads=1]" + ); + m_command_lookup.set_handler( + "stop_mining" + , std::bind(&t_command_parser_executor::stop_mining, &m_parser, p::_1) + , "Stop mining" + ); + m_command_lookup.set_handler( + "print_pool" + , std::bind(&t_command_parser_executor::print_transaction_pool_long, &m_parser, p::_1) + , "Print transaction pool (long format)" + ); + m_command_lookup.set_handler( + "print_pool_sh" + , std::bind(&t_command_parser_executor::print_transaction_pool_short, &m_parser, p::_1) + , "Print transaction pool (short format)" + ); + m_command_lookup.set_handler( + "show_hr" + , std::bind(&t_command_parser_executor::show_hash_rate, &m_parser, p::_1) + , "Start showing hash rate" + ); + m_command_lookup.set_handler( + "hide_hr" + , std::bind(&t_command_parser_executor::hide_hash_rate, &m_parser, p::_1) + , "Stop showing hash rate" + ); + m_command_lookup.set_handler( + "save" + , std::bind(&t_command_parser_executor::save_blockchain, &m_parser, p::_1) + , "Save blockchain" + ); + m_command_lookup.set_handler( + "set_log" + , std::bind(&t_command_parser_executor::set_log_level, &m_parser, p::_1) + , "set_log <level> - Change current log detalization level, <level> is a number 0-4" + ); + m_command_lookup.set_handler( + "diff" + , std::bind(&t_command_parser_executor::show_difficulty, &m_parser, p::_1) + , "Show difficulty" + ); + m_command_lookup.set_handler( + "stop_daemon" + , std::bind(&t_command_parser_executor::stop_daemon, &m_parser, p::_1) + , "Stop the daemon" + ); + m_command_lookup.set_handler( + "exit" + , std::bind(&t_command_parser_executor::stop_daemon, &m_parser, p::_1) + , "Stop the daemon" + ); + m_command_lookup.set_handler( + "print_status" + , std::bind(&t_command_parser_executor::print_status, &m_parser, p::_1) + , "Prints daemon status" + ); + m_command_lookup.set_handler( + "limit" + , std::bind(&t_command_parser_executor::set_limit, &m_parser, p::_1) + , "limit <kB/s> - Set download and upload limit" + ); + m_command_lookup.set_handler( + "limit_up" + , std::bind(&t_command_parser_executor::set_limit_up, &m_parser, p::_1) + , "limit <kB/s> - Set upload limit" + ); + m_command_lookup.set_handler( + "limit_down" + , std::bind(&t_command_parser_executor::set_limit_down, &m_parser, p::_1) + , "limit <kB/s> - Set download limit" + ); + m_command_lookup.set_handler( + "fast_exit" + , std::bind(&t_command_parser_executor::fast_exit, &m_parser, p::_1) + , "Exit" + ); + m_command_lookup.set_handler( + "out_peers" + , std::bind(&t_command_parser_executor::out_peers, &m_parser, p::_1) + , "Set max limit of out peers" + ); + m_command_lookup.set_handler( + "start_save_graph" + , std::bind(&t_command_parser_executor::start_save_graph, &m_parser, p::_1) + , "Start save data for dr monero" + ); + m_command_lookup.set_handler( + "stop_save_graph" + , std::bind(&t_command_parser_executor::stop_save_graph, &m_parser, p::_1) + , "Stop save data for dr monero" + ); +} + +bool t_command_server::process_command_str(const std::string& cmd) +{ + return m_command_lookup.process_command_str(cmd); +} + +bool t_command_server::process_command_vec(const std::vector<std::string>& cmd) +{ + bool result = m_command_lookup.process_command_vec(cmd); + if (!result) + { + help(std::vector<std::string>()); + } + return result; +} + +bool t_command_server::start_handling() +{ + if (m_is_rpc) return false; + + m_command_lookup.start_handling("", get_commands_str()); + + return true; +} + +void t_command_server::stop_handling() +{ + if (m_is_rpc) return; + + m_command_lookup.stop_handling(); +} + +bool t_command_server::help(const std::vector<std::string>& args) +{ + std::cout << get_commands_str() << std::endl; + return true; +} + +std::string t_command_server::get_commands_str() +{ + std::stringstream ss; + ss << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << std::endl; + ss << "Commands: " << std::endl; + std::string usage = m_command_lookup.get_usage(); + boost::replace_all(usage, "\n", "\n "); + usage.insert(0, " "); + ss << usage << std::endl; + return ss.str(); +} + +} // namespace daemonize diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h new file mode 100644 index 000000000..494f44d8b --- /dev/null +++ b/src/daemon/command_server.h @@ -0,0 +1,75 @@ +/** +@file +@details + + +Passing RPC commands: + +@image html images/other/runtime-commands.png + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "console_handler.h" +#include "daemon/command_parser_executor.h" + +namespace daemonize { + +class t_command_server { +private: + t_command_parser_executor m_parser; + epee::console_handlers_binder m_command_lookup; + bool m_is_rpc; + +public: + t_command_server( + uint32_t ip + , uint16_t port + , bool is_rpc = true + , cryptonote::core_rpc_server* rpc_server = NULL + ); + + bool process_command_str(const std::string& cmd); + + bool process_command_vec(const std::vector<std::string>& cmd); + + bool start_handling(); + + void stop_handling(); + +private: + bool help(const std::vector<std::string>& args); + + std::string get_commands_str(); +}; + +} // namespace daemonize diff --git a/src/daemon/core.h b/src/daemon/core.h new file mode 100644 index 000000000..6564e5314 --- /dev/null +++ b/src/daemon/core.h @@ -0,0 +1,99 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cryptonote_core/checkpoints_create.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "misc_log_ex.h" +#include <stdexcept> +#include <boost/program_options.hpp> +#include "daemon/command_line_args.h" + +namespace daemonize +{ + +class t_core final +{ +public: + static void init_options(boost::program_options::options_description & option_spec) + { + cryptonote::core::init_options(option_spec); + cryptonote::miner::init_options(option_spec); + } +private: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + cryptonote::core m_core; + // TEMPORARY HACK - Yes, this creates a copy, but otherwise the original + // variable map could go out of scope before the run method is called + boost::program_options::variables_map const m_vm_HACK; +public: + t_core( + boost::program_options::variables_map const & vm + ) + : m_core{nullptr} + , m_vm_HACK{vm} + { + } + + // TODO - get rid of circular dependencies in internals + void set_protocol(t_protocol_raw & protocol) + { + m_core.set_cryptonote_protocol(&protocol); + } + + void run() + { + //initialize core here + LOG_PRINT_L0("Initializing core..."); + if (!m_core.init(m_vm_HACK)) + { + throw std::runtime_error("Failed to initialize core"); + } + LOG_PRINT_L0("Core initialized OK"); + } + + cryptonote::core & get() + { + return m_core; + } + + ~t_core() + { + LOG_PRINT_L0("Deinitializing core..."); + try { + m_core.deinit(); + m_core.set_cryptonote_protocol(nullptr); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize core..."); + } + } +}; + +} diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index da09acfd4..7931ba03f 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL @@ -25,278 +25,142 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +// // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -// node.cpp : Defines the entry point for the console application. -// Does this file exist? - +#include "daemon/daemon.h" -#include "include_base_utils.h" +#include "common/util.h" +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "daemon/protocol.h" +#include "daemon/rpc.h" +#include "daemon/command_server.h" +#include "misc_log_ex.h" #include "version.h" +#include "../../contrib/epee/include/syncobj.h" using namespace epee; #include <boost/program_options.hpp> +#include <functional> +#include <memory> + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace daemonize { + +struct t_internals { +private: + t_protocol protocol; +public: + t_core core; + t_p2p p2p; + t_rpc rpc; + + t_internals( + boost::program_options::variables_map const & vm + ) + : core{vm} + , protocol{vm, core} + , p2p{vm, protocol} + , rpc{vm, core, p2p} + { + // Handle circular dependencies + protocol.set_p2p_endpoint(p2p.get()); + core.set_protocol(protocol.get()); + } +}; -#include "crypto/hash.h" -#include "console_handler.h" -#include "p2p/net_node.h" -#include "cryptonote_config.h" -#include "cryptonote_core/checkpoints_create.h" -#include "cryptonote_core/checkpoints.h" -#include "cryptonote_core/cryptonote_core.h" -#include "rpc/core_rpc_server.h" -#include "cryptonote_protocol/cryptonote_protocol_handler.h" -#include "daemon_commands_handler.h" -#include "version.h" +void t_daemon::init_options(boost::program_options::options_description & option_spec) +{ + t_core::init_options(option_spec); + t_p2p::init_options(option_spec); + t_rpc::init_options(option_spec); +} -#if defined(WIN32) -#include <crtdbg.h> -#endif +t_daemon::t_daemon( + boost::program_options::variables_map const & vm + ) + : mp_internals{new t_internals{vm}} +{} -namespace po = boost::program_options; +t_daemon::~t_daemon() = default; -namespace +// MSVC is brain-dead and can't default this... +t_daemon::t_daemon(t_daemon && other) { - const command_line::arg_descriptor<std::string> arg_config_file = {"config-file", "Specify configuration file", std::string(CRYPTONOTE_NAME ".conf")}; - const command_line::arg_descriptor<bool> arg_os_version = {"os-version", ""}; - const command_line::arg_descriptor<std::string> arg_log_file = {"log-file", "", ""}; - const command_line::arg_descriptor<int> arg_log_level = {"log-level", "", LOG_LEVEL_0}; - const command_line::arg_descriptor<bool> arg_console = {"no-console", "Disable daemon console commands"}; - const command_line::arg_descriptor<bool> arg_testnet_on = { - "testnet" - , "Run on testnet. The wallet must be launched with --testnet flag." - , false - }; - const command_line::arg_descriptor<bool> arg_dns_checkpoints = {"enforce-dns-checkpointing", "checkpoints from DNS server will be enforced", false}; + if (this != &other) + { + mp_internals = std::move(other.mp_internals); + other.mp_internals.reset(nullptr); + } } -bool command_line_preprocessor(const boost::program_options::variables_map& vm) +// or this +t_daemon & t_daemon::operator=(t_daemon && other) { - bool exit = false; - if (command_line::get_arg(vm, command_line::arg_version)) - { - std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL; - exit = true; - } - if (command_line::get_arg(vm, arg_os_version)) + if (this != &other) { - std::cout << "OS: " << tools::get_os_version_string() << ENDL; - exit = true; + mp_internals = std::move(other.mp_internals); + other.mp_internals.reset(nullptr); } + return *this; +} - if (exit) +bool t_daemon::run(bool interactive) +{ + if (nullptr == mp_internals) { - return true; + throw std::runtime_error{"Can't run stopped daemon"}; } + tools::signal_handler::install(std::bind(&daemonize::t_daemon::stop, this)); - int new_log_level = command_line::get_arg(vm, arg_log_level); - if(new_log_level < LOG_LEVEL_MIN || new_log_level > LOG_LEVEL_MAX) + try { - LOG_PRINT_L0("Wrong log level value: "); - } - else if (log_space::get_set_log_detalisation_level(false) != new_log_level) - { - log_space::get_set_log_detalisation_level(true, new_log_level); - LOG_PRINT_L0("LOG_LEVEL set to " << new_log_level); - } - - return false; -} - -int main(int argc, char* argv[]) -{ - - string_tools::set_module_name_and_folder(argv[0]); -#ifdef WIN32 - _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF ); -#endif - log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); - log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); - LOG_PRINT_L0("Starting..."); + mp_internals->core.run(); + mp_internals->rpc.run(); - TRY_ENTRY(); + daemonize::t_command_server* rpc_commands; - boost::filesystem::path default_data_path {tools::get_default_data_dir()}; - boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + if (interactive) + { + rpc_commands = new daemonize::t_command_server(0, 0, false, mp_internals->rpc.get_server()); + rpc_commands->start_handling(); + } - po::options_description desc_cmd_only("Command line options"); - po::options_description desc_cmd_sett("Command line options and settings options"); + mp_internals->p2p.run(); // blocks until p2p goes down - command_line::add_arg(desc_cmd_only, command_line::arg_help); - command_line::add_arg(desc_cmd_only, command_line::arg_version); - command_line::add_arg(desc_cmd_only, arg_os_version); - // tools::get_default_data_dir() can't be called during static initialization - command_line::add_arg(desc_cmd_only, command_line::arg_data_dir, default_data_path.string()); - command_line::add_arg(desc_cmd_only, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); - command_line::add_arg(desc_cmd_only, arg_config_file); - - command_line::add_arg(desc_cmd_sett, arg_log_file); - command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_console); - command_line::add_arg(desc_cmd_sett, arg_testnet_on); - command_line::add_arg(desc_cmd_sett, arg_dns_checkpoints); - - cryptonote::core::init_options(desc_cmd_sett); - cryptonote::core_rpc_server::init_options(desc_cmd_sett); - nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >::init_options(desc_cmd_sett); - cryptonote::miner::init_options(desc_cmd_sett); - - po::options_description desc_options("Allowed options"); - desc_options.add(desc_cmd_only).add(desc_cmd_sett); - - po::variables_map vm; - bool r = command_line::handle_error_helper(desc_options, [&]() - { - po::store(po::parse_command_line(argc, argv, desc_options), vm); - po::notify(vm); + if (interactive) + { + rpc_commands->stop_handling(); + } + mp_internals->rpc.stop(); + LOG_PRINT("Node stopped.", LOG_LEVEL_0); return true; - }); - if (!r) - return 1; - - if (command_line::get_arg(vm, command_line::arg_help)) - { - std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; - std::cout << desc_options << std::endl; - return false; } - - bool testnet_mode = command_line::get_arg(vm, arg_testnet_on); - - auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; - - std::string data_dir = command_line::get_arg(vm, data_dir_arg); - tools::create_directories_if_necessary(data_dir); - std::string config = command_line::get_arg(vm, arg_config_file); - - boost::filesystem::path data_dir_path(data_dir); - boost::filesystem::path config_path(config); - if (!config_path.has_parent_path()) + catch (std::exception const & ex) { - config_path = data_dir_path / config_path; + LOG_ERROR("Uncaught exception! " << ex.what()); + return false; } - - boost::system::error_code ec; - if (boost::filesystem::exists(config_path, ec)) + catch (...) { - po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), desc_cmd_sett), vm); - } - - //set up logging options - boost::filesystem::path log_file_path(command_line::get_arg(vm, arg_log_file)); - if (log_file_path.empty()) - log_file_path = log_space::log_singletone::get_default_log_file(); - std::string log_dir; - log_dir = log_file_path.has_parent_path() ? log_file_path.parent_path().string() : log_space::log_singletone::get_default_log_folder(); - - log_space::log_singletone::add_logger(LOGGER_FILE, log_file_path.filename().string().c_str(), log_dir.c_str()); - LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); - - if (command_line_preprocessor(vm)) - { - return 0; - } - - LOG_PRINT("Module folder: " << argv[0], LOG_LEVEL_0); - - bool res = true; - cryptonote::checkpoints checkpoints; - res = cryptonote::create_checkpoints(checkpoints); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize checkpoints"); - boost::filesystem::path json(JSON_HASH_FILE_NAME); - boost::filesystem::path checkpoint_json_hashfile_fullpath = data_dir / json; - - //create objects and link them - cryptonote::core ccore(NULL); - - // tell core if we're enforcing dns checkpoints - bool enforce_dns = command_line::get_arg(vm, arg_dns_checkpoints); - ccore.set_enforce_dns_checkpoints(enforce_dns); - - if (testnet_mode) { - LOG_PRINT_L0("Starting in testnet mode!"); - } else { - ccore.set_checkpoints(std::move(checkpoints)); - ccore.set_checkpoints_file_path(checkpoint_json_hashfile_fullpath.string()); + LOG_ERROR("Uncaught exception!"); + return false; } +} - cryptonote::t_cryptonote_protocol_handler<cryptonote::core> cprotocol(ccore, NULL); - nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> > p2psrv { - cprotocol - , testnet_mode ? std::move(config::testnet::NETWORK_ID) : std::move(config::NETWORK_ID) - }; - cryptonote::core_rpc_server rpc_server {ccore, p2psrv, testnet_mode}; - cprotocol.set_p2p_endpoint(&p2psrv); - ccore.set_cryptonote_protocol(&cprotocol); - daemon_cmmands_handler dch(p2psrv, testnet_mode); - - //initialize objects - LOG_PRINT_L0("Initializing P2P server..."); - res = p2psrv.init(vm, testnet_mode); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize P2P server."); - LOG_PRINT_L0("P2P server initialized OK"); - - LOG_PRINT_L0("Initializing protocol..."); - res = cprotocol.init(vm); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize protocol."); - LOG_PRINT_L0("Protocol initialized OK"); - - LOG_PRINT_L0("Initializing core RPC server..."); - res = rpc_server.init(vm); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core RPC server."); - LOG_PRINT_GREEN("Core RPC server initialized OK on port: " << rpc_server.get_binded_port(), LOG_LEVEL_0); - - //initialize core here - LOG_PRINT_L0("Initializing core..."); - res = ccore.init(vm, testnet_mode); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core"); - LOG_PRINT_L0("Core initialized OK"); - - // start components - if(!command_line::has_arg(vm, arg_console)) +void t_daemon::stop() +{ + if (nullptr == mp_internals) { - dch.start_handling(); + throw std::runtime_error{"Can't stop stopped daemon"}; } - - LOG_PRINT_L0("Starting core RPC server..."); - res = rpc_server.run(2, false); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core RPC server."); - LOG_PRINT_L0("Core RPC server started ok"); - - tools::signal_handler::install([&dch, &p2psrv] { - dch.stop_handling(); - p2psrv.send_stop_signal(); - }); - - LOG_PRINT_L0("Starting P2P net loop..."); - p2psrv.run(); - LOG_PRINT_L0("P2P net loop stopped"); - - //stop components - LOG_PRINT_L0("Stopping core rpc server..."); - rpc_server.send_stop_signal(); - rpc_server.timed_wait_server_stop(5000); - - //deinitialize components - LOG_PRINT_L0("Deinitializing core..."); - ccore.deinit(); - LOG_PRINT_L0("Deinitializing RPC server ..."); - rpc_server.deinit(); - LOG_PRINT_L0("Deinitializing protocol..."); - cprotocol.deinit(); - LOG_PRINT_L0("Deinitializing P2P..."); - p2psrv.deinit(); - - - ccore.set_cryptonote_protocol(NULL); - cprotocol.set_p2p_endpoint(NULL); - - LOG_PRINT("Node stopped.", LOG_LEVEL_0); - return 0; - - CATCH_ENTRY_L0("main", 1); + mp_internals->p2p.stop(); + mp_internals->rpc.stop(); + mp_internals.reset(nullptr); // Ensure resources are cleaned up before we return } +} // namespace daemonize diff --git a/src/daemon/daemon.h b/src/daemon/daemon.h new file mode 100644 index 000000000..b0869b25b --- /dev/null +++ b/src/daemon/daemon.h @@ -0,0 +1,54 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <memory> +#include <boost/program_options.hpp> + +namespace daemonize { + +class t_internals; + +class t_daemon final { +public: + static void init_options(boost::program_options::options_description & option_spec); +private: + std::unique_ptr<t_internals> mp_internals; +public: + t_daemon( + boost::program_options::variables_map const & vm + ); + t_daemon(t_daemon && other); + t_daemon & operator=(t_daemon && other); + ~t_daemon(); + + bool run(bool interactive = false); + void stop(); +}; +} diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index 869c2adf7..215cf26de 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -40,22 +40,20 @@ #include "common/util.h" #include "crypto/hash.h" #include "version.h" +#include "../../contrib/otshell_utils/utils.hpp" + +//#include "net/net_helper.h" +//#include "../p2p/p2p_protocol_defs.h" +//#include "../p2p/net_peerlist_boost_serialization.h" +//#include "net/local_ip.h" +//#include "crypto/crypto.h" +//#include "storages/levin_abstract_invoke2.h" -/*! - * \brief I don't really know right now - * - * - */ class daemon_cmmands_handler { nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& m_srv; public: - daemon_cmmands_handler( - nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& srv - , bool testnet - ) - : m_srv(srv) - , m_testnet(testnet) + daemon_cmmands_handler(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& srv):m_srv(srv) { m_cmd_binder.set_handler("help", boost::bind(&daemon_cmmands_handler::help, this, _1), "Show this help"); m_cmd_binder.set_handler("print_pl", boost::bind(&daemon_cmmands_handler::print_pl, this, _1), "Print peer list"); @@ -74,6 +72,14 @@ public: m_cmd_binder.set_handler("save", boost::bind(&daemon_cmmands_handler::save, this, _1), "Save blockchain"); m_cmd_binder.set_handler("set_log", boost::bind(&daemon_cmmands_handler::set_log, this, _1), "set_log <level> - Change current log detalization level, <level> is a number 0-4"); m_cmd_binder.set_handler("diff", boost::bind(&daemon_cmmands_handler::diff, this, _1), "Show difficulty"); + m_cmd_binder.set_handler("out_peers", boost::bind(&daemon_cmmands_handler::out_peers_limit, this, _1), "Set max limit of out peers"); + m_cmd_binder.set_handler("limit_up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit [kB/s]"); + m_cmd_binder.set_handler("limit_down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit [kB/s]"); + m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit [kB/s]"); + m_cmd_binder.set_handler("fast_exit", boost::bind(&daemon_cmmands_handler::fast_exit, this, _1), "Exit"); + m_cmd_binder.set_handler("test_drop_download", boost::bind(&daemon_cmmands_handler::test_drop_download, this, _1), "For network testing, drop downloaded blocks instead checking/adding them to blockchain. Can fake-download blocks very fast."); + m_cmd_binder.set_handler("start_save_graph", boost::bind(&daemon_cmmands_handler::start_save_graph, this, _1), ""); + m_cmd_binder.set_handler("stop_save_graph", boost::bind(&daemon_cmmands_handler::stop_save_graph, this, _1), ""); } bool start_handling() @@ -89,7 +95,7 @@ public: private: epee::srv_console_handlers_binder<nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> > > m_cmd_binder; - bool m_testnet; + //-------------------------------------------------------------------------------- std::string get_commands_str() @@ -122,6 +128,122 @@ private: return true; } //-------------------------------------------------------------------------------- + bool limit_up(const std::vector<std::string>& args) + { + if(args.size()!=1) { + std::cout << "Usage: limit_up <speed>" << ENDL; + return false; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + + //-------------------------------------------------------------------------------- + bool limit_down(const std::vector<std::string>& args) + { + + if(args.size()!=1) { + std::cout << "Usage: limit_down <speed>" << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + +//-------------------------------------------------------------------------------- + bool limit(const std::vector<std::string>& args) + { + if(args.size()!=1) { + std::cout << "Usage: limit_down <speed>" << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + //-------------------------------------------------------------------------------- + bool out_peers_limit(const std::vector<std::string>& args) { + if(args.size()!=1) { + std::cout << "Usage: limit_down <speed>" << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + return false; + } + + LOG_PRINT_RED_L0("connections_count: " << limit); + m_srv.m_config.m_net_config.connections_count = limit; + return true; + } + //-------------------------------------------------------------------------------- bool show_hr(const std::vector<std::string>& args) { if(!m_srv.get_payload_object().get_core().get_miner().is_mining()) @@ -234,7 +356,13 @@ private: return true; } + // TODO what the hell causes compilation warning in following code line +PUSH_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) log_space::log_singletone::get_set_log_detalisation_level(true, l); + int otshell_utils_log_level = 100 - (l * 25); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); +POP_WARNINGS return true; } @@ -374,7 +502,7 @@ private: } cryptonote::account_public_address adr; - if(!cryptonote::get_account_address_from_str(adr, m_testnet, args.front())) + if(!cryptonote::get_account_address_from_str(adr, args.front())) { std::cout << "target account address has wrong format" << std::endl; return true; diff --git a/src/daemon/executor.cpp b/src/daemon/executor.cpp new file mode 100644 index 000000000..bb188c174 --- /dev/null +++ b/src/daemon/executor.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "daemon/executor.h" + +#include "misc_log_ex.h" + +#include "common/command_line.h" +#include "cryptonote_config.h" +#include "version.h" + +#include <string> + +namespace daemonize +{ + std::string const t_executor::NAME = "Monero Daemon"; + + void t_executor::init_options( + boost::program_options::options_description & configurable_options + ) + { + t_daemon::init_options(configurable_options); + } + + std::string const & t_executor::name() + { + return NAME; + } + + t_daemon t_executor::create_daemon( + boost::program_options::variables_map const & vm + ) + { + LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); + return t_daemon{vm}; + } + + bool t_executor::run_interactive( + boost::program_options::variables_map const & vm + ) + { + epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + return t_daemon{vm}.run(true); + } +} + diff --git a/src/daemon/executor.h b/src/daemon/executor.h new file mode 100644 index 000000000..553896875 --- /dev/null +++ b/src/daemon/executor.h @@ -0,0 +1,60 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "daemon/daemon.h" +#include <boost/program_options/options_description.hpp> +#include <boost/program_options/variables_map.hpp> +#include <string> +#include <vector> + +namespace daemonize +{ + class t_executor final + { + public: + typedef ::daemonize::t_daemon t_daemon; + + static std::string const NAME; + + static void init_options( + boost::program_options::options_description & configurable_options + ); + + std::string const & name(); + + t_daemon create_daemon( + boost::program_options::variables_map const & vm + ); + + bool run_interactive( + boost::program_options::variables_map const & vm + ); + }; +} diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp new file mode 100644 index 000000000..62b655ccd --- /dev/null +++ b/src/daemon/main.cpp @@ -0,0 +1,265 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "common/command_line.h" +#include "common/scoped_message_writer.h" +#include "common/util.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/miner.h" +#include "daemon/command_server.h" +#include "daemon/daemon.h" +#include "daemon/executor.h" +#include "daemonizer/daemonizer.h" +#include "misc_log_ex.h" +#include "p2p/net_node.h" +#include "rpc/core_rpc_server.h" +#include <boost/program_options.hpp> +#include "daemon/command_line_args.h" +#include "blockchain_db/db_types.h" + +namespace po = boost::program_options; +namespace bf = boost::filesystem; + +int main(int argc, char const * argv[]) +{ + try { + + _note_c("dbg/main", "Begin of main()"); + // TODO parse the debug options like set log level right here at start + + epee::string_tools::set_module_name_and_folder(argv[0]); + + // Build argument description + po::options_description all_options("All"); + po::options_description hidden_options("Hidden"); + po::options_description visible_options("Options"); + po::options_description core_settings("Settings"); + po::positional_options_description positional_options; + { + bf::path default_data_dir = daemonizer::get_default_data_dir(); + bf::path default_testnet_data_dir = {default_data_dir / "testnet"}; + + // Misc Options + + command_line::add_arg(visible_options, command_line::arg_help); + command_line::add_arg(visible_options, command_line::arg_version); + command_line::add_arg(visible_options, daemon_args::arg_os_version); + command_line::add_arg(visible_options, command_line::arg_data_dir, default_data_dir.string()); + command_line::add_arg(visible_options, command_line::arg_testnet_data_dir, default_testnet_data_dir.string()); + bf::path default_conf = default_data_dir / std::string(CRYPTONOTE_NAME ".conf"); + command_line::add_arg(visible_options, daemon_args::arg_config_file, default_conf.string()); + command_line::add_arg(visible_options, command_line::arg_test_drop_download); + command_line::add_arg(visible_options, command_line::arg_test_dbg_lock_sleep); + command_line::add_arg(visible_options, command_line::arg_test_drop_download_height); + + // Settings + bf::path default_log = default_data_dir / std::string(CRYPTONOTE_NAME ".log"); + command_line::add_arg(core_settings, daemon_args::arg_log_file, default_log.string()); + command_line::add_arg(core_settings, daemon_args::arg_log_level); + command_line::add_arg(core_settings, daemon_args::arg_testnet_on); + command_line::add_arg(core_settings, daemon_args::arg_dns_checkpoints); + command_line::add_arg(core_settings, daemon_args::arg_db_type); + daemonizer::init_options(hidden_options, visible_options); + daemonize::t_executor::init_options(core_settings); + + // Hidden options + command_line::add_arg(hidden_options, daemon_args::arg_command); + + visible_options.add(core_settings); + all_options.add(visible_options); + all_options.add(hidden_options); + + // Positional + positional_options.add(daemon_args::arg_command.name, -1); // -1 for unlimited arguments + } + + // Do command line parsing + po::variables_map vm; + bool ok = command_line::handle_error_helper(visible_options, [&]() + { + boost::program_options::store( + boost::program_options::command_line_parser(argc, argv) + .options(all_options).positional(positional_options).run() + , vm + ); + + return true; + }); + if (!ok) return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << "Usage: " + std::string{argv[0]} + " [options|settings] [daemon_command...]" << std::endl << std::endl; + std::cout << visible_options << std::endl; + return 0; + } + + // Monero Version + if (command_line::get_arg(vm, command_line::arg_version)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL; + return 0; + } + + // OS + if (command_line::get_arg(vm, daemon_args::arg_os_version)) + { + std::cout << "OS: " << tools::get_os_version_string() << ENDL; + return 0; + } + + epee::g_test_dbg_lock_sleep = command_line::get_arg(vm, command_line::arg_test_dbg_lock_sleep); + + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + // verify that blockchaindb type is valid + if(cryptonote::blockchain_db_types.count(db_type) == 0) + { + std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl; + for (const auto& type : cryptonote::blockchain_db_types) + { + std::cout << "\t" << type << std::endl; + } + return 0; + } + + bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on); + + auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + + // Create data dir if it doesn't exist + boost::filesystem::path data_dir = boost::filesystem::absolute( + command_line::get_arg(vm, data_dir_arg)); + tools::create_directories_if_necessary(data_dir.string()); + + // FIXME: not sure on windows implementation default, needs further review + //bf::path relative_path_base = daemonizer::get_relative_path_base(vm); + bf::path relative_path_base = data_dir; + + std::string config = command_line::get_arg(vm, daemon_args::arg_config_file); + + boost::filesystem::path data_dir_path(data_dir); + boost::filesystem::path config_path(config); + if (!config_path.has_parent_path()) + { + config_path = data_dir / config_path; + } + + boost::system::error_code ec; + if (bf::exists(config_path, ec)) + { + po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), core_settings), vm); + } + po::notify(vm); + + // If there are positional options, we're running a daemon command + { + auto command = command_line::get_arg(vm, daemon_args::arg_command); + + if (command.size()) + { + auto rpc_ip_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_ip); + auto rpc_port_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + if (testnet_mode) + { + rpc_port_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_testnet_rpc_bind_port); + } + + uint32_t rpc_ip; + uint16_t rpc_port; + if (!epee::string_tools::get_ip_int32_from_string(rpc_ip, rpc_ip_str)) + { + std::cerr << "Invalid IP: " << rpc_ip_str << std::endl; + return 1; + } + if (!epee::string_tools::get_xtype_from_string(rpc_port, rpc_port_str)) + { + std::cerr << "Invalid port: " << rpc_port_str << std::endl; + return 1; + } + + daemonize::t_command_server rpc_commands{rpc_ip, rpc_port}; + if (rpc_commands.process_command_vec(command)) + { + return 0; + } + else + { + std::cerr << "Unknown command" << std::endl; + return 1; + } + } + } + + // Start with log level 0 + epee::log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); + + // Set log level + { + int new_log_level = command_line::get_arg(vm, daemon_args::arg_log_level); + if(new_log_level < LOG_LEVEL_MIN || new_log_level > LOG_LEVEL_MAX) + { + LOG_PRINT_L0("Wrong log level value: " << new_log_level); + } + else if (epee::log_space::get_set_log_detalisation_level(false) != new_log_level) + { + epee::log_space::get_set_log_detalisation_level(true, new_log_level); + int otshell_utils_log_level = 100 - (new_log_level * 20); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); + LOG_PRINT_L0("LOG_LEVEL set to " << new_log_level); + } + } + + // Set log file + { + bf::path log_file_path{bf::absolute(command_line::get_arg(vm, daemon_args::arg_log_file), relative_path_base)}; + + epee::log_space::log_singletone::add_logger( + LOGGER_FILE + , log_file_path.filename().string().c_str() + , log_file_path.parent_path().string().c_str() + ); + } + + _note_c("dbg/main", "Moving from main() into the daemonize now."); + + return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm); + } + catch (std::exception const & ex) + { + LOG_ERROR("Exception in main! " << ex.what()); + } + catch (...) + { + LOG_ERROR("Exception in main!"); + } + return 1; +} diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h new file mode 100644 index 000000000..355a784b1 --- /dev/null +++ b/src/daemon/p2p.h @@ -0,0 +1,99 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "daemon/protocol.h" +#include "misc_log_ex.h" +#include "p2p/net_node.h" +#include <stdexcept> +#include <boost/program_options.hpp> + +namespace daemonize +{ + +class t_p2p final +{ +private: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + typedef nodetool::node_server<t_protocol_raw> t_node_server; +public: + static void init_options(boost::program_options::options_description & option_spec) + { + t_node_server::init_options(option_spec); + } +private: + t_node_server m_server; +public: + t_p2p( + boost::program_options::variables_map const & vm + , t_protocol & protocol + ) + : m_server{protocol.get()} + { + //initialize objects + LOG_PRINT_L0("Initializing p2p server..."); + if (!m_server.init(vm)) + { + throw std::runtime_error("Failed to initialize p2p server."); + } + LOG_PRINT_L0("P2p server initialized OK"); + } + + t_node_server & get() + { + return m_server; + } + + void run() + { + LOG_PRINT_L0("Starting p2p net loop..."); + m_server.run(); + LOG_PRINT_L0("p2p net loop stopped"); + } + + void stop() + { + m_server.send_stop_signal(); + } + + ~t_p2p() + { + LOG_PRINT_L0("Deinitializing p2p..."); + try { + m_server.deinit(); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize p2p..."); + } + } +}; + +} diff --git a/src/daemon/protocol.h b/src/daemon/protocol.h new file mode 100644 index 000000000..2f6a66f49 --- /dev/null +++ b/src/daemon/protocol.h @@ -0,0 +1,88 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "misc_log_ex.h" +#include "p2p/net_node.h" +#include <stdexcept> +#include <boost/program_options.hpp> + +namespace daemonize +{ + +class t_protocol final +{ +private: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + typedef nodetool::node_server<t_protocol_raw> t_node_server; + + t_protocol_raw m_protocol; +public: + t_protocol( + boost::program_options::variables_map const & vm + , t_core & core + ) + : m_protocol{core.get(), nullptr} + { + LOG_PRINT_L0("Initializing cryptonote protocol..."); + if (!m_protocol.init(vm)) + { + throw std::runtime_error("Failed to initialize cryptonote protocol."); + } + LOG_PRINT_L0("Cryptonote protocol initialized OK"); + } + + t_protocol_raw & get() + { + return m_protocol; + } + + void set_p2p_endpoint( + t_node_server & server + ) + { + m_protocol.set_p2p_endpoint(&server); + } + + ~t_protocol() + { + LOG_PRINT_L0("Deinitializing cryptonote_protocol..."); + try { + m_protocol.deinit(); + m_protocol.set_p2p_endpoint(nullptr); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize protocol..."); + } + } +}; + +} diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h new file mode 100644 index 000000000..6477f440e --- /dev/null +++ b/src/daemon/rpc.h @@ -0,0 +1,101 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "misc_log_ex.h" +#include "rpc/core_rpc_server.h" +#include <boost/program_options.hpp> +#include <stdexcept> + +namespace daemonize +{ + +class t_rpc final +{ +public: + static void init_options(boost::program_options::options_description & option_spec) + { + cryptonote::core_rpc_server::init_options(option_spec); + } +private: + cryptonote::core_rpc_server m_server; +public: + t_rpc( + boost::program_options::variables_map const & vm + , t_core & core + , t_p2p & p2p + ) + : m_server{core.get(), p2p.get()} + { + LOG_PRINT_L0("Initializing core rpc server..."); + if (!m_server.init(vm)) + { + throw std::runtime_error("Failed to initialize core rpc server."); + } + LOG_PRINT_GREEN("Core rpc server initialized OK on port: " << m_server.get_binded_port(), LOG_LEVEL_0); + } + + void run() + { + LOG_PRINT_L0("Starting core rpc server..."); + if (!m_server.run(2, false)) + { + throw std::runtime_error("Failed to start core rpc server."); + } + LOG_PRINT_L0("Core rpc server started ok"); + } + + void stop() + { + LOG_PRINT_L0("Stopping core rpc server..."); + m_server.send_stop_signal(); + m_server.timed_wait_server_stop(5000); + } + + cryptonote::core_rpc_server* get_server() + { + return &m_server; + } + + ~t_rpc() + { + LOG_PRINT_L0("Deinitializing rpc server..."); + try { + m_server.deinit(); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize rpc server..."); + } + } +}; + +} diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp new file mode 100644 index 000000000..8ef91600c --- /dev/null +++ b/src/daemon/rpc_command_executor.cpp @@ -0,0 +1,872 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "string_tools.h" +#include "common/scoped_message_writer.h" +#include "daemon/rpc_command_executor.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "cryptonote_core/cryptonote_core.h" +#include <boost/format.hpp> +#include <ctime> +#include <string> + +namespace daemonize { + +namespace { + void print_peer(std::string const & prefix, cryptonote::peer const & peer) + { + time_t now; + time(&now); + time_t last_seen = static_cast<time_t>(peer.last_seen); + + std::string id_str; + std::string port_str; + std::string elapsed = epee::misc_utils::get_time_interval_string(now - last_seen); + std::string ip_str = epee::string_tools::get_ip_string_from_int32(peer.ip); + epee::string_tools::xtype_to_string(peer.id, id_str); + epee::string_tools::xtype_to_string(peer.port, port_str); + std::string addr_str = ip_str + ":" + port_str; + tools::msg_writer() << boost::format("%-10s %-25s %-25s %s") % prefix % id_str % addr_str % elapsed; + } + + void print_block_header(cryptonote::block_header_responce const & header) + { + tools::success_msg_writer() + << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << std::endl + << "previous hash: " << header.prev_hash << std::endl + << "nonce: " << boost::lexical_cast<std::string>(header.nonce) << std::endl + << "is orphan: " << header.orphan_status << std::endl + << "height: " << boost::lexical_cast<std::string>(header.height) << std::endl + << "depth: " << boost::lexical_cast<std::string>(header.depth) << std::endl + << "hash: " << header.hash + << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl + << "reward: " << boost::lexical_cast<std::string>(header.reward); + } +} + +t_rpc_command_executor::t_rpc_command_executor( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server + ) + : m_rpc_client(NULL), m_rpc_server(rpc_server) +{ + if (is_rpc) + { + m_rpc_client = new tools::t_rpc_client(ip, port); + } + else + { + if (rpc_server == NULL) + { + throw std::runtime_error("If not calling commands via RPC, rpc_server pointer must be non-null"); + } + } + + m_is_rpc = is_rpc; +} + +t_rpc_command_executor::~t_rpc_command_executor() +{ + if (m_rpc_client != NULL) + { + delete m_rpc_client; + } +} + +bool t_rpc_command_executor::print_peer_list() { + cryptonote::COMMAND_RPC_GET_PEER_LIST::request req; + cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; + + std::string failure_message = "Couldn't retrieve peer list"; + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/get_peer_list", failure_message.c_str())) + { + return false; + } + } + else + { + if (!m_rpc_server->on_get_peer_list(req, res)) + { + tools::fail_msg_writer() << failure_message; + return false; + } + } + + for (auto & peer : res.white_list) + { + print_peer("white", peer); + } + + for (auto & peer : res.gray_list) + { + print_peer("gray", peer); + } + + return true; +} + +bool t_rpc_command_executor::save_blockchain() { + cryptonote::COMMAND_RPC_SAVE_BC::request req; + cryptonote::COMMAND_RPC_SAVE_BC::response res; + + std::string fail_message = "Couldn't save blockchain"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/save_bc", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_save_bc(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + } + } + + tools::success_msg_writer() << "Blockchain saved"; + + return true; +} + +bool t_rpc_command_executor::show_hash_rate() { + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::request req; + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::response res; + req.visible = true; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_log_hash_rate", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_log_hash_rate(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + } + } + + tools::success_msg_writer() << "Hash rate logging is on"; + + return true; +} + +bool t_rpc_command_executor::hide_hash_rate() { + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::request req; + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::response res; + req.visible = false; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_log_hash_rate", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_log_hash_rate(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Hash rate logging is off"; + + return true; +} + +bool t_rpc_command_executor::show_difficulty() { + cryptonote::COMMAND_RPC_GET_INFO::request req; + cryptonote::COMMAND_RPC_GET_INFO::response res; + + std::string fail_message = "Problem fetching info"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/getinfo", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_info(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "BH: " << res.height + << ", DIFF: " << res.difficulty + << ", HR: " << (int) res.difficulty / 60L << " H/s"; + + return true; +} + +bool t_rpc_command_executor::print_connections() { + cryptonote::COMMAND_RPC_GET_CONNECTIONS::request req; + cryptonote::COMMAND_RPC_GET_CONNECTIONS::response res; + epee::json_rpc::error error_resp; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "/get_connections", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_connections(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::msg_writer() << std::setw(30) << std::left << "Remote Host" + << std::setw(20) << "Peer id" + << std::setw(30) << "Recv/Sent (inactive,sec)" + << std::setw(25) << "State" + << std::setw(20) << "Livetime(sec)" + << std::setw(12) << "Down (kB/s)" + << std::setw(14) << "Down(now)" + << std::setw(10) << "Up (kB/s)" + << std::setw(13) << "Up(now)" + << std::endl; + + for (auto & info : res.connections) + { + std::string address = info.incoming ? "INC " : "OUT "; + address += info.ip + ":" + info.port; + //std::string in_out = info.incoming ? "INC " : "OUT "; + tools::msg_writer() + //<< std::setw(30) << std::left << in_out + << std::setw(30) << std::left << address + << std::setw(20) << info.peer_id + << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" + << std::setw(25) << info.state + << std::setw(20) << info.live_time + << std::setw(12) << info.avg_download + << std::setw(14) << info.current_download + << std::setw(10) << info.avg_upload + << std::setw(13) << info.current_upload + + << std::left << (info.localhost ? "[LOCALHOST]" : "") + << std::left << (info.local_ip ? "[LAN]" : ""); + //tools::msg_writer() << boost::format("%-25s peer_id: %-25s %s") % address % info.peer_id % in_out; + + } + + return true; +} + +bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index) { + +// this function appears to not exist in the json rpc api, and so is commented +// until such a time as it does. + +/* + cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request req; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response res; + epee::json_rpc::error error_resp; + + req.start_height = start_block_index; + req.end_height = end_block_index; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "getblockheadersrange", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_getblockheadersrange(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + for (auto & header : res.headers) + { + std::cout + << "major version: " << header.major_version << std::endl + << "minor version: " << header.minor_version << std::endl + << "height: " << header.height << ", timestamp: " << header.timestamp << ", difficulty: " << header.difficulty << std::endl + << "block id: " << header.hash << std::endl + << "previous block id: " << header.prev_hash << std::endl + << "difficulty: " << header.difficulty << ", nonce " << header.nonce << std::endl; + } + +*/ + return true; +} + +bool t_rpc_command_executor::set_log_level(int8_t level) { + cryptonote::COMMAND_RPC_SET_LOG_LEVEL::request req; + cryptonote::COMMAND_RPC_SET_LOG_LEVEL::response res; + req.level = level; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_log_level", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_log_level(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Log level is now " << std::to_string(level); + + return true; +} + +bool t_rpc_command_executor::print_height() { + cryptonote::COMMAND_RPC_GET_HEIGHT::request req; + cryptonote::COMMAND_RPC_GET_HEIGHT::response res; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/getheight", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_height(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << boost::lexical_cast<std::string>(res.height); + + return true; +} + +bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request req; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response res; + epee::json_rpc::error error_resp; + + req.hash = epee::string_tools::pod_to_hex(block_hash); + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "getblockheaderbyhash", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_block_header_by_hash(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + print_block_header(res.block_header); + + return true; +} + +bool t_rpc_command_executor::print_block_by_height(uint64_t height) { + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request req; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response res; + epee::json_rpc::error error_resp; + + req.height = height; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "getblockheaderbyheight", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_block_header_by_height(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + print_block_header(res.block_header); + + return true; +} + +bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) { + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res; + + std::string fail_message = "Problem fetching transaction"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/gettransactions", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_transactions(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + if (1 == res.txs_as_hex.size()) + { + tools::success_msg_writer() << res.txs_as_hex.front(); + } + else + { + tools::fail_msg_writer() << "transaction wasn't found: <" << transaction_hash << '>' << std::endl; + } + + return true; +} + +bool t_rpc_command_executor::print_transaction_pool_long() { + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res; + + std::string fail_message = "Problem fetching transaction pool"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/get_transaction_pool", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_transaction_pool(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + if (res.transactions.empty() && res.spent_key_images.empty()) + { + tools::msg_writer() << "Pool is empty" << std::endl; + } + if (! res.transactions.empty()) + { + tools::msg_writer() << "Transactions: "; + for (auto & tx_info : res.transactions) + { + tools::msg_writer() << "id: " << tx_info.id_hash << std::endl + << tx_info.tx_json << std::endl + << "blob_size: " << tx_info.blob_size << std::endl + << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl + << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl + << "max_used_block_height: " << tx_info.max_used_block_height << std::endl + << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl + << "last_failed_height: " << tx_info.last_failed_height << std::endl + << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl; + } + if (res.spent_key_images.empty()) + { + tools::msg_writer() << "WARNING: Inconsistent pool state - no spent key images"; + } + } + if (! res.spent_key_images.empty()) + { + tools::msg_writer() << ""; // one newline + tools::msg_writer() << "Spent key images: "; + for (const cryptonote::spent_key_image_info& kinfo : res.spent_key_images) + { + tools::msg_writer() << "key image: " << kinfo.id_hash; + if (kinfo.txs_hashes.size() == 1) + { + tools::msg_writer() << " tx: " << kinfo.txs_hashes[0]; + } + else if (kinfo.txs_hashes.size() == 0) + { + tools::msg_writer() << " WARNING: spent key image has no txs associated"; + } + else + { + tools::msg_writer() << " NOTE: key image for multiple txs: " << kinfo.txs_hashes.size(); + for (const std::string& tx_id : kinfo.txs_hashes) + { + tools::msg_writer() << " tx: " << tx_id; + } + } + } + if (res.transactions.empty()) + { + tools::msg_writer() << "WARNING: Inconsistent pool state - no transactions"; + } + } + + return true; +} + +bool t_rpc_command_executor::print_transaction_pool_short() { + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res; + + std::string fail_message = "Problem fetching transaction pool"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/get_transaction_pool", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_transaction_pool(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + if (res.transactions.empty()) + { + tools::msg_writer() << "Pool is empty" << std::endl; + } + for (auto & tx_info : res.transactions) + { + tools::msg_writer() << "id: " << tx_info.id_hash << std::endl + << "blob_size: " << tx_info.blob_size << std::endl + << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl + << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl + << "max_used_block_height: " << tx_info.max_used_block_height << std::endl + << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl + << "last_failed_height: " << tx_info.last_failed_height << std::endl + << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl; + } + + return true; +} + +// TODO: update this for testnet +bool t_rpc_command_executor::start_mining(cryptonote::account_public_address address, uint64_t num_threads) { + cryptonote::COMMAND_RPC_START_MINING::request req; + cryptonote::COMMAND_RPC_START_MINING::response res; + req.miner_address = cryptonote::get_account_address_as_str(false, address); + req.threads_count = num_threads; + + std::string fail_message = "Mining did not start"; + + if (m_is_rpc) + { + if (m_rpc_client->rpc_request(req, res, "/start_mining", fail_message.c_str())) + { + tools::success_msg_writer() << "Mining started"; + } + } + else + { + if (!m_rpc_server->on_start_mining(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::stop_mining() { + cryptonote::COMMAND_RPC_STOP_MINING::request req; + cryptonote::COMMAND_RPC_STOP_MINING::response res; + + std::string fail_message = "Mining did not stop"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/stop_mining", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_stop_mining(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Mining stopped"; + return true; +} + +bool t_rpc_command_executor::stop_daemon() +{ + cryptonote::COMMAND_RPC_STOP_DAEMON::request req; + cryptonote::COMMAND_RPC_STOP_DAEMON::response res; + +//# ifdef WIN32 +// // Stop via service API +// // TODO - this is only temporary! Get rid of hard-coded constants! +// bool ok = windows::stop_service("BitMonero Daemon"); +// ok = windows::uninstall_service("BitMonero Daemon"); +// //bool ok = windows::stop_service(SERVICE_NAME); +// //ok = windows::uninstall_service(SERVICE_NAME); +// if (ok) +// { +// return true; +// } +//# endif + + // Stop via RPC + std::string fail_message = "Daemon did not stop"; + + if (m_is_rpc) + { + if(!m_rpc_client->rpc_request(req, res, "/stop_daemon", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_stop_daemon(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Stop signal sent"; + + return true; +} + +bool t_rpc_command_executor::print_status() +{ + if (!m_is_rpc) + { + tools::success_msg_writer() << "print_status makes no sense in interactive mode"; + return true; + } + + bool daemon_is_alive = m_rpc_client->check_connection(); + + if(daemon_is_alive) { + tools::success_msg_writer() << "bitmonerod is running"; + } + else { + tools::fail_msg_writer() << "bitmonerod is NOT running"; + } + + return true; +} + +bool t_rpc_command_executor::set_limit(int limit) +{ + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + return true; +} + +bool t_rpc_command_executor::set_limit_up(int limit) +{ + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + return true; +} + +bool t_rpc_command_executor::set_limit_down(int limit) +{ + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + return true; +} + +bool t_rpc_command_executor::fast_exit() +{ + cryptonote::COMMAND_RPC_FAST_EXIT::request req; + cryptonote::COMMAND_RPC_FAST_EXIT::response res; + + std::string fail_message = "Daemon did not stop"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/fast_exit", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_fast_exit(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Daemon stopped"; + return true; +} + +bool t_rpc_command_executor::out_peers(uint64_t limit) +{ + cryptonote::COMMAND_RPC_OUT_PEERS::request req; + cryptonote::COMMAND_RPC_OUT_PEERS::response res; + + epee::json_rpc::error error_resp; + + req.out_peers = limit; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "/out_peers", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_out_peers(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::start_save_graph() +{ + cryptonote::COMMAND_RPC_START_SAVE_GRAPH::request req; + cryptonote::COMMAND_RPC_START_SAVE_GRAPH::response res; + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/start_save_graph", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_start_save_graph(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::stop_save_graph() +{ + cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::request req; + cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::response res; + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/stop_save_graph", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_stop_save_graph(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + return true; +} + +}// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h new file mode 100644 index 000000000..43b8a9fe0 --- /dev/null +++ b/src/daemon/rpc_command_executor.h @@ -0,0 +1,117 @@ +/** +@file +@details + +@image html images/other/runtime-commands.png + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "common/rpc_client.h" +#include "misc_log_ex.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "p2p/net_node.h" +#include "rpc/core_rpc_server.h" + +namespace daemonize { + +class t_rpc_command_executor final { +private: + tools::t_rpc_client* m_rpc_client; + cryptonote::core_rpc_server* m_rpc_server; + bool m_is_rpc; + +public: + t_rpc_command_executor( + uint32_t ip + , uint16_t port + , bool is_rpc = true + , cryptonote::core_rpc_server* rpc_server = NULL + ); + + ~t_rpc_command_executor(); + + bool print_peer_list(); + + bool save_blockchain(); + + bool show_hash_rate(); + + bool hide_hash_rate(); + + bool show_difficulty(); + + bool print_connections(); + + bool print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index); + + bool set_log_level(int8_t level); + + bool print_height(); + + bool print_block_by_hash(crypto::hash block_hash); + + bool print_block_by_height(uint64_t height); + + bool print_transaction(crypto::hash transaction_hash); + + bool print_transaction_pool_long(); + + bool print_transaction_pool_short(); + + bool start_mining(cryptonote::account_public_address address, uint64_t num_threads); + + bool stop_mining(); + + bool stop_daemon(); + + bool print_status(); + + bool set_limit(int limit); + + bool set_limit_up(int limit); + + bool set_limit_down(int limit); + + bool fast_exit(); + + bool out_peers(uint64_t limit); + + bool start_save_graph(); + + bool stop_save_graph(); +}; + +} // namespace daemonize diff --git a/src/daemonizer/CMakeLists.txt b/src/daemonizer/CMakeLists.txt new file mode 100644 index 000000000..d5830111c --- /dev/null +++ b/src/daemonizer/CMakeLists.txt @@ -0,0 +1,74 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if(MSVC OR MINGW) + set(daemonizer_sources + windows_service.cpp + windows_daemonizer.inl + ) +else() + set(daemonizer_sources + posix_fork.cpp + posix_daemonizer.inl + ) +endif() + +set(daemonizer_headers +) + +if(MSVC OR MINGW) + set(daemonizer_private_headers + daemonizer.h + windows_service.h + windows_service_runner.h + ) +else() + set(daemonizer_private_headers + daemonizer.h + posix_fork.h + ) +endif() + +bitmonero_private_headers(daemonizer + ${daemonizer_private_headers}) +bitmonero_add_library(daemonizer + ${daemonizer_sources} + ${daemonizer_headers} + ${daemonizer_private_headers}) +target_link_libraries(daemonizer + LINK_PRIVATE + common + ${Boost_CHRONO_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_REGEX_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${UPNP_LIBRARIES} + ${EXTRA_LIBRARIES}) diff --git a/src/daemonizer/daemonizer.h b/src/daemonizer/daemonizer.h new file mode 100644 index 000000000..6097a58f6 --- /dev/null +++ b/src/daemonizer/daemonizer.h @@ -0,0 +1,38 @@ +#pragma once + +#include <boost/filesystem/path.hpp> +#include <boost/program_options/options_description.hpp> +#include <boost/program_options/variables_map.hpp> + +namespace daemonizer +{ + void init_options( + boost::program_options::options_description & hidden_options + , boost::program_options::options_description & normal_options + ); + + boost::filesystem::path get_default_data_dir(); + + boost::filesystem::path get_relative_path_base( + boost::program_options::variables_map const & vm + ); + + /** + * @arg create_before_detach - this indicates that the daemon should be + * created before the fork, giving it a chance to report initialization + * errors. At the time of this writing, this is not possible in the primary + * daemon (likely due to the size of the blockchain in memory). + */ + template <typename T_executor> + bool daemonize( + int argc, char const * argv[] + , T_executor && executor // universal ref + , boost::program_options::variables_map const & vm + ); +} + +#ifdef WIN32 +# include "daemonizer/windows_daemonizer.inl" +#else +# include "daemonizer/posix_daemonizer.inl" +#endif diff --git a/src/daemonizer/posix_daemonizer.inl b/src/daemonizer/posix_daemonizer.inl new file mode 100644 index 000000000..e06d43d61 --- /dev/null +++ b/src/daemonizer/posix_daemonizer.inl @@ -0,0 +1,60 @@ +#pragma once + +#include "common/scoped_message_writer.h" +#include "common/util.h" +#include "daemonizer/posix_fork.h" + +#include <boost/filesystem/operations.hpp> +#include <boost/filesystem/path.hpp> + +namespace daemonizer +{ + namespace + { + const command_line::arg_descriptor<bool> arg_detach = { + "detach" + , "Run as daemon" + }; + } + + inline void init_options( + boost::program_options::options_description & hidden_options + , boost::program_options::options_description & normal_options + ) + { + command_line::add_arg(normal_options, arg_detach); + } + + inline boost::filesystem::path get_default_data_dir() + { + return boost::filesystem::absolute(tools::get_default_data_dir()); + } + + inline boost::filesystem::path get_relative_path_base( + boost::program_options::variables_map const & vm + ) + { + return boost::filesystem::current_path(); + } + + template <typename T_executor> + inline bool daemonize( + int argc, char const * argv[] + , T_executor && executor // universal ref + , boost::program_options::variables_map const & vm + ) + { + if (command_line::has_arg(vm, arg_detach)) + { + auto daemon = executor.create_daemon(vm); + tools::success_msg_writer() << "Forking to background..."; + posix::fork(); + return daemon.run(); + } + else + { + //LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); + return executor.run_interactive(vm); + } + } +} diff --git a/src/daemonizer/posix_fork.cpp b/src/daemonizer/posix_fork.cpp new file mode 100644 index 000000000..c068912ec --- /dev/null +++ b/src/daemonizer/posix_fork.cpp @@ -0,0 +1,108 @@ +// Copyright (c) 2003-2011 Christopher M. Kohlhoff (chris at kohlhoff dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include "daemonizer/posix_fork.h" +#include "misc_log_ex.h" + +#include <cstdlib> +#include <fcntl.h> +#include <unistd.h> +#include <stdexcept> +#include <string> +#include <sys/stat.h> + +namespace posix { + +namespace { + void quit(std::string const & message) + { + LOG_ERROR(message); + throw std::runtime_error(message); + } +} + +void fork() +{ + // Fork the process and have the parent exit. If the process was started + // from a shell, this returns control to the user. Forking a new process is + // also a prerequisite for the subsequent call to setsid(). + if (pid_t pid = ::fork()) + { + if (pid > 0) + { + // We're in the parent process and need to exit. + // + // When the exit() function is used, the program terminates without + // invoking local variables' destructors. Only global variables are + // destroyed. + exit(0); + } + else + { + quit("First fork failed"); + } + } + + // Make the process a new session leader. This detaches it from the + // terminal. + setsid(); + + // A process inherits its working directory from its parent. This could be + // on a mounted filesystem, which means that the running daemon would + // prevent this filesystem from being unmounted. Changing to the root + // directory avoids this problem. + if (chdir("/") < 0) + { + quit("Unable to change working directory to root"); + } + + // The file mode creation mask is also inherited from the parent process. + // We don't want to restrict the permissions on files created by the + // daemon, so the mask is cleared. + umask(0); + + // A second fork ensures the process cannot acquire a controlling terminal. + if (pid_t pid = ::fork()) + { + if (pid > 0) + { + exit(0); + } + else + { + quit("Second fork failed"); + } + } + + // Close the standard streams. This decouples the daemon from the terminal + // that started it. + close(0); + close(1); + close(2); + + // We don't want the daemon to have any standard input. + if (open("/dev/null", O_RDONLY) < 0) + { + quit("Unable to open /dev/null"); + } + + // Send standard output to a log file. + const char* output = "/tmp/bitmonero.daemon.stdout.stderr"; + const int flags = O_WRONLY | O_CREAT | O_APPEND; + const mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; + if (open(output, flags, mode) < 0) + { + quit("Unable to open output file: " + std::string(output)); + } + + // Also send standard error to the same log file. + if (dup(1) < 0) + { + quit("Unable to dup output descriptor"); + } +} + +} // namespace posix diff --git a/src/daemonizer/posix_fork.h b/src/daemonizer/posix_fork.h new file mode 100644 index 000000000..f099685e9 --- /dev/null +++ b/src/daemonizer/posix_fork.h @@ -0,0 +1,11 @@ +#pragma once + +#ifndef WIN32 + +namespace posix { + +void fork(); + +} + +#endif diff --git a/src/daemonizer/windows_daemonizer.inl b/src/daemonizer/windows_daemonizer.inl new file mode 100644 index 000000000..c099f4097 --- /dev/null +++ b/src/daemonizer/windows_daemonizer.inl @@ -0,0 +1,156 @@ +#pragma once + +#include "common/util.h" +#include "daemonizer/windows_service.h" +#include "daemonizer/windows_service_runner.h" + +#include <shlobj.h> +#include <boost/filesystem/operations.hpp> +#include <boost/filesystem/path.hpp> + +namespace daemonizer +{ + namespace + { + const command_line::arg_descriptor<bool> arg_install_service = { + "install-service" + , "Install Windows service" + }; + const command_line::arg_descriptor<bool> arg_uninstall_service = { + "uninstall-service" + , "Uninstall Windows service" + }; + const command_line::arg_descriptor<bool> arg_start_service = { + "start-service" + , "Start Windows service" + }; + const command_line::arg_descriptor<bool> arg_stop_service = { + "stop-service" + , "Stop Windows service" + }; + const command_line::arg_descriptor<bool> arg_is_service = { + "run-as-service" + , "Hidden -- true if running as windows service" + }; + + std::string get_argument_string(int argc, char const * argv[]) + { + std::string result = ""; + for (int i = 1; i < argc; ++i) + { + result += " " + std::string{argv[i]}; + } + return result; + } + } + + inline void init_options( + boost::program_options::options_description & hidden_options + , boost::program_options::options_description & normal_options + ) + { + command_line::add_arg(normal_options, arg_install_service); + command_line::add_arg(normal_options, arg_uninstall_service); + command_line::add_arg(normal_options, arg_start_service); + command_line::add_arg(normal_options, arg_stop_service); + command_line::add_arg(hidden_options, arg_is_service); + } + + inline boost::filesystem::path get_default_data_dir() + { + bool admin; + if (!windows::check_admin(admin)) + { + admin = false; + } + if (admin) + { + return boost::filesystem::absolute( + tools::get_special_folder_path(CSIDL_COMMON_APPDATA, true) + "\\" + CRYPTONOTE_NAME + ); + } + else + { + return boost::filesystem::absolute( + tools::get_special_folder_path(CSIDL_APPDATA, true) + "\\" + CRYPTONOTE_NAME + ); + } + } + + inline boost::filesystem::path get_relative_path_base( + boost::program_options::variables_map const & vm + ) + { + if (command_line::has_arg(vm, arg_is_service)) + { + if (command_line::has_arg(vm, command_line::arg_data_dir)) + { + return command_line::get_arg(vm, command_line::arg_data_dir); + } + else + { + return tools::get_default_data_dir(); + } + } + else + { + return boost::filesystem::current_path(); + } + } + + template <typename T_executor> + inline bool daemonize( + int argc, char const * argv[] + , T_executor && executor // universal ref + , boost::program_options::variables_map const & vm + ) + { + std::string arguments = get_argument_string(argc, argv); + + if (command_line::has_arg(vm, arg_is_service)) + { + // TODO - Set the service status here for return codes + windows::t_service_runner<typename T_executor::t_daemon>::run( + executor.name() + , executor.create_daemon(vm) + ); + return true; + } + else if (command_line::has_arg(vm, arg_install_service)) + { + if (windows::ensure_admin(arguments)) + { + arguments += " --run-as-service"; + return windows::install_service(executor.name(), arguments); + } + } + else if (command_line::has_arg(vm, arg_uninstall_service)) + { + if (windows::ensure_admin(arguments)) + { + return windows::uninstall_service(executor.name()); + } + } + else if (command_line::has_arg(vm, arg_start_service)) + { + if (windows::ensure_admin(arguments)) + { + return windows::start_service(executor.name()); + } + } + else if (command_line::has_arg(vm, arg_stop_service)) + { + if (windows::ensure_admin(arguments)) + { + return windows::stop_service(executor.name()); + } + } + else // interactive + { + //LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); + return executor.run_interactive(vm); + } + + return false; + } +} diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp new file mode 100644 index 000000000..1b0ee2ef4 --- /dev/null +++ b/src/daemonizer/windows_service.cpp @@ -0,0 +1,341 @@ +#undef UNICODE +#undef _UNICODE + +#include "common/scoped_message_writer.h" +#include "daemonizer/windows_service.h" +#include "string_tools.h" +#include <chrono> +#include <iostream> +#include <utility> +#include <memory> +#include <shellapi.h> +#include <thread> +#include <windows.h> + +namespace windows { + +namespace { + typedef std::unique_ptr<std::remove_pointer<SC_HANDLE>::type, decltype(&::CloseServiceHandle)> service_handle; + + std::string get_last_error() + { + LPSTR p_error_text = nullptr; + + FormatMessage( + FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_IGNORE_INSERTS + , nullptr + , GetLastError() + , MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT) + , reinterpret_cast<LPSTR>(&p_error_text) + , 0 + , nullptr + ); + + if (nullptr == p_error_text) + { + return ""; + } + else + { + return std::string{p_error_text}; + LocalFree(p_error_text); + } + } + + bool relaunch_as_admin( + std::string const & command + , std::string const & arguments + ) + { + SHELLEXECUTEINFO info{}; + info.cbSize = sizeof(info); + info.lpVerb = "runas"; + info.lpFile = command.c_str(); + info.lpParameters = arguments.c_str(); + info.hwnd = nullptr; + info.nShow = SW_SHOWNORMAL; + if (!ShellExecuteEx(&info)) + { + tools::fail_msg_writer() << "Admin relaunch failed: " << get_last_error(); + return false; + } + else + { + return true; + } + } + + // When we relaunch as admin, Windows opens a new window. This just pauses + // to allow the user to read any output. + void pause_to_display_admin_window_messages() + { + std::chrono::milliseconds how_long{1500}; + std::this_thread::sleep_for(how_long); + } +} + +bool check_admin(bool & result) +{ + BOOL is_admin = FALSE; + PSID p_administrators_group = nullptr; + + SID_IDENTIFIER_AUTHORITY nt_authority = SECURITY_NT_AUTHORITY; + + if (!AllocateAndInitializeSid( + &nt_authority + , 2 + , SECURITY_BUILTIN_DOMAIN_RID + , DOMAIN_ALIAS_RID_ADMINS + , 0, 0, 0, 0, 0, 0 + , &p_administrators_group + )) + { + tools::fail_msg_writer() << "Security Identifier creation failed: " << get_last_error(); + return false; + } + + if (!CheckTokenMembership( + nullptr + , p_administrators_group + , &is_admin + )) + { + tools::fail_msg_writer() << "Permissions check failed: " << get_last_error(); + return false; + } + + result = is_admin ? true : false; + + return true; +} + +bool ensure_admin( + std::string const & arguments + ) +{ + bool admin; + + if (!check_admin(admin)) + { + return false; + } + + if (admin) + { + return true; + } + else + { + std::string command = epee::string_tools::get_current_module_path(); + relaunch_as_admin(command, arguments); + return false; + } +} + +bool install_service( + std::string const & service_name + , std::string const & arguments + ) +{ + std::string command = epee::string_tools::get_current_module_path(); + std::string full_command = command + arguments; + + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT | SC_MANAGER_CREATE_SERVICE + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + CreateService( + p_manager.get() + , service_name.c_str() + , service_name.c_str() + , 0 + //, GENERIC_EXECUTE | GENERIC_READ + , SERVICE_WIN32_OWN_PROCESS + , SERVICE_DEMAND_START + , SERVICE_ERROR_NORMAL + , full_command.c_str() + , nullptr + , nullptr + , "" + //, "NT AUTHORITY\\LocalService" + , nullptr // Implies LocalSystem account + , nullptr + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't create service: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service installed"; + + pause_to_display_admin_window_messages(); + + return true; +} + +bool start_service( + std::string const & service_name + ) +{ + tools::msg_writer() << "Starting service"; + + SERVICE_STATUS_PROCESS service_status = {}; + DWORD unused = 0; + + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + OpenService( + p_manager.get() + , service_name.c_str() + //, SERVICE_START | SERVICE_QUERY_STATUS + , SERVICE_START + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't find service: " << get_last_error(); + return false; + } + + if (!StartService( + p_service.get() + , 0 + , nullptr + )) + { + tools::fail_msg_writer() << "Service start request failed: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service started"; + + pause_to_display_admin_window_messages(); + + return true; +} + +bool stop_service( + std::string const & service_name + ) +{ + tools::msg_writer() << "Stopping service"; + + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + OpenService( + p_manager.get() + , service_name.c_str() + , SERVICE_STOP | SERVICE_QUERY_STATUS + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't find service: " << get_last_error(); + return false; + } + + SERVICE_STATUS status = {}; + if (!ControlService(p_service.get(), SERVICE_CONTROL_STOP, &status)) + { + tools::fail_msg_writer() << "Couldn't request service stop: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service stopped"; + + pause_to_display_admin_window_messages(); + + return true; +} + +bool uninstall_service( + std::string const & service_name + ) +{ + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + OpenService( + p_manager.get() + , service_name.c_str() + , SERVICE_QUERY_STATUS | DELETE + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't find service: " << get_last_error(); + return false; + } + + SERVICE_STATUS status = {}; + if (!DeleteService(p_service.get())) + { + tools::fail_msg_writer() << "Couldn't uninstall service: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service uninstalled"; + + pause_to_display_admin_window_messages(); + + return true; +} + +} // namespace windows diff --git a/src/daemonizer/windows_service.h b/src/daemonizer/windows_service.h new file mode 100644 index 000000000..11f5fdcdc --- /dev/null +++ b/src/daemonizer/windows_service.h @@ -0,0 +1,36 @@ +#pragma once + +#ifdef WIN32 + +#undef UNICODE +#undef _UNICODE + +#include <string> +#include <windows.h> + +namespace windows +{ + bool check_admin(bool & result); + + bool ensure_admin( + std::string const & arguments + ); + + bool install_service( + std::string const & service_name + , std::string const & arguments + ); + + bool uninstall_service( + std::string const & service_name + ); + + bool start_service( + std::string const & service_name + ); + + bool stop_service( + std::string const & service_name + ); +} +#endif diff --git a/src/daemonizer/windows_service_runner.h b/src/daemonizer/windows_service_runner.h new file mode 100644 index 000000000..79b10b136 --- /dev/null +++ b/src/daemonizer/windows_service_runner.h @@ -0,0 +1,157 @@ +#pragma once + +#ifdef WIN32 + +#undef UNICODE +#undef _UNICODE + +#include "daemonizer/windows_service.h" +#include <memory> +#include <string> +#include <vector> +#include <windows.h> + +namespace windows { + namespace + { + std::vector<char> vecstring(std::string const & str) + { + std::vector<char> result{str.begin(), str.end()}; + result.push_back('\0'); + return result; + } + } + + template <typename T_handler> + class t_service_runner final + { + private: + SERVICE_STATUS_HANDLE m_status_handle{nullptr}; + SERVICE_STATUS m_status{}; + std::mutex m_lock{}; + std::string m_name; + T_handler m_handler; + + static std::unique_ptr<t_service_runner<T_handler>> sp_instance; + public: + t_service_runner( + std::string name + , T_handler handler + ) + : m_name{std::move(name)} + , m_handler{std::move(handler)} + { + m_status.dwServiceType = SERVICE_WIN32; + m_status.dwCurrentState = SERVICE_STOPPED; + m_status.dwControlsAccepted = 0; + m_status.dwWin32ExitCode = NO_ERROR; + m_status.dwServiceSpecificExitCode = NO_ERROR; + m_status.dwCheckPoint = 0; + m_status.dwWaitHint = 0; + } + + t_service_runner & operator=(t_service_runner && other) + { + if (this != &other) + { + m_status_handle = std::move(other.m_status_handle); + m_status = std::move(other.m_status); + m_name = std::move(other.m_name); + m_handler = std::move(other.m_handler); + } + return *this; + } + + static void run( + std::string name + , T_handler handler + ) + { + sp_instance.reset(new t_service_runner<T_handler>{ + std::move(name) + , std::move(handler) + }); + + sp_instance->run_(); + } + + private: + void run_() + { + SERVICE_TABLE_ENTRY table[] = + { + { vecstring(m_name).data(), &service_main } + , { 0, 0 } + }; + + StartServiceCtrlDispatcher(table); + } + + void report_status(DWORD status) + { + m_status.dwCurrentState = status; + if (status == SERVICE_RUNNING) + { + m_status.dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN; + } + else if(status == SERVICE_STOP_PENDING) + { + m_status.dwControlsAccepted = 0; + } + SetServiceStatus(m_status_handle, &m_status); + } + + static void WINAPI service_main(DWORD argc, LPSTR * argv) + { + sp_instance->service_main_(argc, argv); + } + + void service_main_(DWORD argc, LPSTR * argv) + { + m_status_handle = RegisterServiceCtrlHandler(m_name.c_str(), &on_state_change_request); + if (m_status_handle == nullptr) return; + + report_status(SERVICE_START_PENDING); + + report_status(SERVICE_RUNNING); + + m_handler.run(); + + on_state_change_request_(SERVICE_CONTROL_STOP); + + // Ensure that the service is uninstalled + uninstall_service(m_name); + } + + static void WINAPI on_state_change_request(DWORD control_code) + { + sp_instance->on_state_change_request_(control_code); + } + + void on_state_change_request_(DWORD control_code) + { + switch (control_code) + { + case SERVICE_CONTROL_INTERROGATE: + break; + case SERVICE_CONTROL_SHUTDOWN: + case SERVICE_CONTROL_STOP: + report_status(SERVICE_STOP_PENDING); + m_handler.stop(); + report_status(SERVICE_STOPPED); + break; + case SERVICE_CONTROL_PAUSE: + break; + case SERVICE_CONTROL_CONTINUE: + break; + default: + break; + } + } + }; + + template <typename T_handler> + std::unique_ptr<t_service_runner<T_handler>> t_service_runner<T_handler>::sp_instance; +} + +#endif diff --git a/src/miner/CMakeLists.txt b/src/miner/CMakeLists.txt index 83bda57cc..2a55c88f7 100644 --- a/src/miner/CMakeLists.txt +++ b/src/miner/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/miner/simpleminer.cpp b/src/miner/simpleminer.cpp index 3223bdd50..6212f88f5 100644 --- a/src/miner/simpleminer.cpp +++ b/src/miner/simpleminer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -41,6 +41,7 @@ using namespace epee; namespace po = boost::program_options; +unsigned int epee::g_test_dbg_lock_sleep = 0; int main(int argc, char** argv) { diff --git a/src/miner/simpleminer.h b/src/miner/simpleminer.h index f6897e20e..9b8e84b3e 100644 --- a/src/miner/simpleminer.h +++ b/src/miner/simpleminer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/miner/simpleminer_protocol_defs.h b/src/miner/simpleminer_protocol_defs.h index 22ea958e7..c63246578 100644 --- a/src/miner/simpleminer_protocol_defs.h +++ b/src/miner/simpleminer_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/miner/target_helper.h b/src/miner/target_helper.h index 723d61a3c..0214b8038 100644 --- a/src/miner/target_helper.h +++ b/src/miner/target_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/CMakeLists.txt b/src/mnemonics/CMakeLists.txt index 66ef4f3f1..ce9520956 100644 --- a/src/mnemonics/CMakeLists.txt +++ b/src/mnemonics/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 7258b8a15..590320550 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index b75de30e9..5cde7fb02 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/english.h b/src/mnemonics/english.h index 8ca0a3fe3..fee428817 100644 --- a/src/mnemonics/english.h +++ b/src/mnemonics/english.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
@@ -728,7 +728,7 @@ namespace Language "initiate",
"injury",
"inkling",
- "incline",
+ "inline",
"inmate",
"innocent",
"inorganic",
@@ -1073,7 +1073,7 @@ namespace Language "ouch",
"ought",
"ounce",
- "launchpad",
+ "ourselves",
"oust",
"outbreak",
"oval",
diff --git a/src/mnemonics/japanese.h b/src/mnemonics/japanese.h index fc3d1ee74..5f7adce18 100644 --- a/src/mnemonics/japanese.h +++ b/src/mnemonics/japanese.h @@ -1,6 +1,6 @@ // Word list originally created by dabura667
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 06815e39d..0e7d60a02 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/old_english.h b/src/mnemonics/old_english.h index b91a593b6..f7974f4d5 100644 --- a/src/mnemonics/old_english.h +++ b/src/mnemonics/old_english.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/portuguese.h b/src/mnemonics/portuguese.h index 6a90659e0..f60991a4d 100644 --- a/src/mnemonics/portuguese.h +++ b/src/mnemonics/portuguese.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/singleton.h b/src/mnemonics/singleton.h index 74e121e4e..ab4e95cf2 100644 --- a/src/mnemonics/singleton.h +++ b/src/mnemonics/singleton.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/spanish.h b/src/mnemonics/spanish.h index f71627086..b638ec1b0 100644 --- a/src/mnemonics/spanish.h +++ b/src/mnemonics/spanish.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
@@ -1690,4 +1690,4 @@ namespace Language };
}
-#endif
\ No newline at end of file +#endif diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt new file mode 100644 index 000000000..541b90fa9 --- /dev/null +++ b/src/p2p/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2014, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +cmake_minimum_required (VERSION 2.6) +project (bitmonero CXX) + +file(GLOB P2P *) +source_group(p2p FILES ${P2P}) + +#add_library(p2p ${P2P}) + +#bitmonero_private_headers(p2p ${P2P}) +bitmonero_add_library(p2p ${P2P}) +#target_link_libraries(p2p) +# LINK_PRIVATE +# ${Boost_CHRONO_LIBRARY} +# ${Boost_REGEX_LIBRARY} +# ${Boost_SYSTEM_LIBRARY} +# ${Boost_THREAD_LIBRARY} +# ${EXTRA_LIBRARIES}) +add_dependencies(p2p + version) diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp new file mode 100644 index 000000000..0454f30e3 --- /dev/null +++ b/src/p2p/connection_basic.cpp @@ -0,0 +1,299 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#include "connection_basic.hpp" + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include <boost/filesystem.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <fstream> +#include <sstream> +#include <iomanip> +#include <algorithm> +#include <mutex> + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> +#include "../../contrib/epee/include/net/abstract_tcp_server2.h" + +#include "../../contrib/otshell_utils/utils.hpp" +#include "data_logger.hpp" +using namespace nOT::nUtils; + +// TODO: +#include "../../src/p2p/network_throttle-detail.hpp" +#include "../../src/cryptonote_core/cryptonote_core.h" + +// ################################################################################################ +// local (TU local) headers +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + + std::string to_string(t_connection_type type) + { + if (type == e_connection_type_NET) + return std::string("NET"); + else if (type == e_connection_type_RPC) + return std::string("RPC"); + else if (type == e_connection_type_P2P) + return std::string("P2P"); + + return std::string("UNKNOWN"); + } + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + int m_peer_number; // e.g. for debug/stats +}; + + +} // namespace +} // namespace + +// ################################################################################################ +// The implementation part +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// connection_basic_pimpl +// ================================================================================================ + +connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name) { } + +// ================================================================================================ +// connection_basic +// ================================================================================================ + +// static variables: +int connection_basic_pimpl::m_default_tos; + +// methods: +connection_basic::connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number) + : + mI( new connection_basic_pimpl("peer") ), + strand_(io_service), + socket_(io_service), + m_want_close_connection(false), + m_was_shutdown(false), + m_ref_sock_count(ref_sock_count) +{ + ++ref_sock_count; // increase the global counter + mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number + + string remote_addr_str = "?"; + try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ; + + _note("Spawned connection p2p#"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count); + //boost::filesystem::create_directories("log/dr-monero/net/"); +} + +connection_basic::~connection_basic() { + string remote_addr_str = "?"; + try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ; + _note("Destructing connection p2p#"<<mI->m_peer_number << " to " << remote_addr_str); +} + +void connection_basic::set_rate_up_limit(uint64_t limit) { + + // TODO remove __SCALING_FACTOR... + const double SCALING_FACTOR = 2.1; // to acheve the best performance + limit *= SCALING_FACTOR; + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().set_target_speed(limit); + network_throttle_manager::get_global_throttle_out().set_real_target_speed(limit / SCALING_FACTOR); + } + save_limit_to_file(limit); +} + +void connection_basic::set_rate_down_limit(uint64_t limit) { + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().set_target_speed(limit); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().set_target_speed(limit); + } + save_limit_to_file(limit); +} + + +void connection_basic::save_limit_to_file(int limit) { + // saving limit to file + if (!epee::net_utils::data_logger::m_save_graph) + return; + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + epee::net_utils::data_logger::get_instance().add_data("upload_limit", network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + epee::net_utils::data_logger::get_instance().add_data("download_limit", network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024); + } +} + +void connection_basic::set_tos_flag(int tos) { + connection_basic_pimpl::m_default_tos = tos; +} + +int connection_basic::get_tos_flag() { + return connection_basic_pimpl::m_default_tos; +} + +void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q_len) { + double delay=0; // will be calculated + do + { // rate limiting + if (m_was_shutdown) { + _dbg2("m_was_shutdown - so abort sleep"); + return; + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + delay *= 0.50; + if (delay > 0) { + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // debug sleep + _dbg1("sleep in sleep_before_packet"); + epee::net_utils::data_logger::get_instance().add_data("sleep_up", ms); + boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); + } + } while(delay > 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_exact( packet_size * 700); // increase counter - global + } + +} +void connection_basic::set_start_time() { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds(); +} + +void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) { + sleep_before_packet(cb,1,-1); + _info_c("net/out/size", "handler_write (direct) - before ASIO write, for packet="<<cb<<" B (after sleep)"); + set_start_time(); +} + +void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) { + sleep_before_packet(cb,2,q_len); + _info_c("net/out/size", "handler_write (after write, from queue="<<q_len<<") - before ASIO write, for packet="<<cb<<" B (after sleep)"); + + set_start_time(); +} + +void connection_basic::logger_handle_net_read(size_t size) { // network data read + size /= 1024; + epee::net_utils::data_logger::get_instance().add_data("download", size); +} + +void connection_basic::logger_handle_net_write(size_t size) { + size /= 1024; + epee::net_utils::data_logger::get_instance().add_data("upload", size); +} + +double connection_basic::get_sleep_time(size_t cb) { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_out); + auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb); + return t; +} + +void connection_basic::set_save_graph(bool save_graph) { + epee::net_utils::data_logger::m_save_graph = save_graph; +} + + +} // namespace +} // namespace + diff --git a/src/p2p/connection_basic.hpp b/src/p2p/connection_basic.hpp new file mode 100644 index 000000000..d8101afe4 --- /dev/null +++ b/src/p2p/connection_basic.hpp @@ -0,0 +1,140 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// ! This file might contain variable names same as in template class connection<> +// ! from files contrib/epee/include/net/abstract_tcp_server2.* +// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;) +// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part) +// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as: + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#ifndef INCLUDED_p2p_connection_basic_hpp +#define INCLUDED_p2p_connection_basic_hpp + + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/syncobj.h" + +namespace epee +{ +namespace net_utils +{ + + /************************************************************************/ + /* */ + /************************************************************************/ + /// Represents a single connection from a client. + +class connection_basic_pimpl; // PIMPL for this class + + enum t_connection_type { // type of the connection (of this server), e.g. so that we will know how to limit it + e_connection_type_NET = 0, // default (not used?) + e_connection_type_RPC = 1, // the rpc commands (probably not rate limited, not chunked, etc) + e_connection_type_P2P = 2 // to other p2p node (probably limited) + }; + + std::string to_string(t_connection_type type); + +class connection_basic { // not-templated base class for rapid developmet of some code parts + public: + std::unique_ptr< connection_basic_pimpl > mI; // my Implementation + + // moved here from orginal connecton<> - common member variables that do not depend on template in connection<> + volatile uint32_t m_want_close_connection; + std::atomic<bool> m_was_shutdown; + critical_section m_send_que_lock; + std::list<std::string> m_send_que; + volatile bool m_is_multithreaded; + double m_start_time; + /// Strand to ensure the connection's handlers are not called concurrently. + boost::asio::io_service::strand strand_; + /// Socket for the connection. + boost::asio::ip::tcp::socket socket_; + + std::atomic<long> &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/-- + public: + // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator + connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number); + + virtual ~connection_basic(); + + // various handlers to be called from connection class: + void do_send_handler_write(const void * ptr , size_t cb); + void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part + + void logger_handle_net_write(size_t size); // network data written + void logger_handle_net_read(size_t size); // network data read + + void set_start_time(); + + // config for rate limit + + static void set_rate_up_limit(uint64_t limit); + static void set_rate_down_limit(uint64_t limit); + + // config misc + static void set_tos_flag(int tos); // ToS / QoS flag + static int get_tos_flag(); + + // handlers and sleep + void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) + static void save_limit_to_file(int limit); ///< for dr-monero + static double get_sleep_time(size_t cb); + + static void set_save_graph(bool save_graph); +}; + +} // nameserver +} // nameserver + +#endif + + diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp new file mode 100644 index 000000000..54fd33e82 --- /dev/null +++ b/src/p2p/data_logger.cpp @@ -0,0 +1,173 @@ +#include "data_logger.hpp" +#include <stdexcept> + +#include <boost/chrono.hpp> +#include <boost/filesystem.hpp> +#include <chrono> +#include "../../contrib/otshell_utils/utils.hpp" + +namespace epee +{ +namespace net_utils +{ + data_logger &data_logger::get_instance() { + std::call_once(m_singleton, + [] { + _info_c("dbg/data","Creating singleton of data_logger"); + if (m_state != data_logger_state::state_before_init) { _erro_c("dbg/data","Internal error in singleton"); throw std::runtime_error("data_logger singleton"); } + m_state = data_logger_state::state_during_init; + m_obj.reset(new data_logger()); + m_state = data_logger_state::state_ready_to_use; + } + ); + + if (m_state != data_logger_state::state_ready_to_use) { + _erro ("trying to use not working data_logger"); + throw std::runtime_error("data_logger ctor state"); + } + + return * m_obj; + } + + data_logger::data_logger() { + _note_c("dbg/data","Starting data logger (for graphs data)"); + if (m_state != data_logger_state::state_during_init) { _erro_c("dbg/data","Singleton ctor state"); throw std::runtime_error("data_logger ctor state"); } + std::lock_guard<std::mutex> lock(mMutex); // lock + + // prepare all the files for given data channels: + mFilesMap["peers"] = data_logger::fileData("log/dr-monero/peers.data"); + mFilesMap["download"] = data_logger::fileData("log/dr-monero/net/in-all.data"); + mFilesMap["upload"] = data_logger::fileData("log/dr-monero/net/out-all.data"); + mFilesMap["request"] = data_logger::fileData("log/dr-monero/net/req-all.data"); + mFilesMap["sleep_down"] = data_logger::fileData("log/dr-monero/down_sleep_log.data"); + mFilesMap["sleep_up"] = data_logger::fileData("log/dr-monero/up_sleep_log.data"); + mFilesMap["calc_time"] = data_logger::fileData("log/dr-monero/get_objects_calc_time.data"); + mFilesMap["blockchain_processing_time"] = data_logger::fileData("log/dr-monero/blockchain_log.data"); + mFilesMap["block_processing"] = data_logger::fileData("log/dr-monero/block_proc.data"); + + mFilesMap["peers_limit"] = data_logger::fileData("log/dr-monero/peers_limit.info"); + mFilesMap["download_limit"] = data_logger::fileData("log/dr-monero/limit_down.info"); + mFilesMap["upload_limit"] = data_logger::fileData("log/dr-monero/limit_up.info"); + + mFilesMap["peers_limit"].mLimitFile = true; + mFilesMap["download_limit"].mLimitFile = true; + mFilesMap["upload_limit"].mLimitFile = true; + + // do NOT modify mFilesMap below this point, since there is no locking for this used (yet) + + _info_c("dbg/data","Creating thread for data logger"); // create timer thread + m_thread_maybe_running=true; + std::shared_ptr<std::thread> logger_thread(new std::thread([&]() { + _info_c("dbg/data","Inside thread for data logger"); + while (m_state == data_logger_state::state_during_init) { // wait for creation to be done (in other thread, in singleton) before actually running + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + _info_c("dbg/data","Inside thread for data logger - going into main loop"); + while (m_state == data_logger_state::state_ready_to_use) { // run as long as we are not closing the single object + std::this_thread::sleep_for(std::chrono::seconds(1)); + saveToFile(); // save all the pending data + } + _info_c("dbg/data","Inside thread for data logger - done the main loop"); + m_thread_maybe_running=false; + })); + logger_thread->detach(); + _info_c("dbg/data","Data logger constructed"); + } + + data_logger::~data_logger() { + _note_c("dbg/data","Destructor of the data logger"); + { + std::lock_guard<std::mutex> lock(mMutex); + m_state = data_logger_state::state_dying; + } + _info_c("dbg/data","State was set to dying"); + while(m_thread_maybe_running) { // wait for the thread to exit + std::this_thread::sleep_for(std::chrono::seconds(1)); + _info_c("dbg/data","Waiting for background thread to exit"); + } + _info_c("dbg/data","Thread exited"); + } + + void data_logger::kill_instance() { + m_state = data_logger_state::state_dying; + m_obj.reset(); + } + + void data_logger::add_data(std::string filename, unsigned int data) { + std::lock_guard<std::mutex> lock(mMutex); + if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } + + if (mFilesMap.find(filename) == mFilesMap.end()) { // no such file/counter + _erro_c("dbg/data","Trying to use not opened data file filename="<<filename); + _erro_c("dbg/data","Disabling saving of graphs due to error"); + m_save_graph=false; // <--- disabling saving graphs + return; + } + + if (mFilesMap[filename].mLimitFile) { // this holds a number (that is not additive) - e.g. the limit setting + mFilesMap[filename].mDataToSave = data; + } else { + mFilesMap[filename].mDataToSave += data; // this holds a number that should be sum of all accumulated samples + } + } + + bool data_logger::is_dying() { + if (m_state == data_logger_state::state_dying) { + return true; + } + else { + return false; + } + } + + void data_logger::saveToFile() { + _dbg2_c("dbg/data","saving to files"); + std::lock_guard<std::mutex> lock(mMutex); + if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } + nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/"); + for (auto &element : mFilesMap) + { + element.second.save(); + if (!element.second.mLimitFile) element.second.mDataToSave = 0; + } + } + + // the inner class: + + double data_logger::fileData::get_current_time() { + #if defined(__APPLE__) + auto point = std::chrono::system_clock::now(); + #else + auto point = std::chrono::steady_clock::now(); + #endif + auto time_from_epoh = point.time_since_epoch(); + auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + return ms_f / 1000.; + } + + data_logger::fileData::fileData(std::string pFile) { + _dbg3_c("dbg/data","opening data file named pFile="<<pFile<<" for this="<<this); + mFile = std::make_shared<std::ofstream> (pFile); + _dbg1_c("dbg/data","opened data file named pFile="<<pFile<<" in mFile="<<mFile<<" for this="<<this); + mPath = pFile; + } + + void data_logger::fileData::save() { + if (!data_logger::m_save_graph) return; // <--- disabled + _dbg2_c("dbg/data","saving to the file now, mFile="<<mFile); + mFile->open(mPath, std::ios::app); + *mFile << static_cast<int>(get_current_time()) << " " << mDataToSave << std::endl; + mFile->close(); + } + + +data_logger_state data_logger::m_state(data_logger_state::state_before_init); ///< (static) state of the singleton object +std::atomic<bool> data_logger::m_save_graph(false); // (static) +std::atomic<bool> data_logger::m_thread_maybe_running(false); // (static) +std::once_flag data_logger::m_singleton; // (static) +std::unique_ptr<data_logger> data_logger::m_obj; // (static) + +} // namespace +} // namespace + diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp new file mode 100644 index 000000000..f38dacdcb --- /dev/null +++ b/src/p2p/data_logger.hpp @@ -0,0 +1,76 @@ +#ifndef INCLUDED_p2p_data_logger_hpp +#define INCLUDED_p2p_data_logger_hpp + +#include <string> +#include <map> +#include <fstream> +#include <memory> +#include <thread> +#include <mutex> +#include <atomic> + +namespace epee +{ +namespace net_utils +{ + +enum class data_logger_state { state_before_init, state_during_init, state_ready_to_use, state_dying }; + +/*** +@note: use it ONLY via singleton! It will be spawned then, and will auto destruct on program exit. +@note: do call ::kill_instance() before exiting main, at end of main. But before make sure no one else (e.g. no other threads) will try to use this/singleton +@note: it is not allowed to use this class from code "runnig before or after main", e.g. from ctors of static objects, because of static-creation-order races +@note: on creation (e.g. from singleton), it spawns a thread that saves all data in background +*/ + class data_logger { + public: + static data_logger &get_instance(); ///< singleton + static void kill_instance(); ///< call this before ending main to allow more gracefull shutdown of the main singleton and it's background thread + ~data_logger(); ///< destr, will be called when singleton is killed when global m_obj dies. will kill theads etc + + private: + data_logger(); ///< constructor is private, use only via singleton get_instance + + public: + data_logger(const data_logger &ob) = delete; // use only one per program + data_logger(data_logger &&ob) = delete; + data_logger & operator=(const data_logger&) = delete; + data_logger & operator=(data_logger&&) = delete; + + void add_data(std::string filename, unsigned int data); ///< use this to append data here. Use it only the singleton. It locks itself. + + static std::atomic<bool> m_save_graph; ///< global setting flag, should we save all the data or not (can disable logging graphs data) + static bool is_dying(); + + private: + static std::once_flag m_singleton; ///< to guarantee singleton creates the object exactly once + static data_logger_state m_state; ///< state of the singleton object + static std::atomic<bool> m_thread_maybe_running; ///< is the background thread (more or less) running, or is it fully finished + static std::unique_ptr<data_logger> m_obj; ///< the singleton object. Only use it via get_instance(). Can be killed by kill_instance() + + /*** + * one graph/file with data + */ + class fileData { + public: + fileData() = default; + fileData(const fileData &ob) = delete; + fileData(std::string pFile); + + std::shared_ptr<std::ofstream> mFile; + long int mDataToSave = 0; ///< sum of the data (in current interval, will be counted from 0 on next interval) + static double get_current_time(); + void save(); + std::string mPath; + bool mLimitFile = false; ///< this holds a number (that is not additive) - e.g. the limit setting + }; + + std::map<std::string, fileData> mFilesMap; + std::mutex mMutex; + void saveToFile(); ///< write data to the target files. do not use this directly + }; + +} // namespace +} // namespace + +#endif diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 220bda0c2..df726703f 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -80,14 +80,15 @@ namespace nodetool public: typedef t_payload_net_handler payload_net_handler; - node_server( - t_payload_net_handler& payload_handler - , boost::uuids::uuid network_id - ) - : m_payload_handler(payload_handler) - , m_allow_local_ip(false) - , m_hide_my_port(false) - , m_network_id(std::move(network_id)) + node_server(t_payload_net_handler& payload_handler) + :m_payload_handler(payload_handler), + m_current_number_of_out_peers(0), + m_allow_local_ip(false), + m_hide_my_port(false), + m_no_igd(false), + m_save_graph(false), + is_closing(false), + m_net_server( epee::net_utils::e_connection_type_P2P ) // this is a P2P connection of the main p2p node server, because this is class node_server<> {} virtual ~node_server() {} @@ -95,7 +96,7 @@ namespace nodetool static void init_options(boost::program_options::options_description& desc); bool run(); - bool init(const boost::program_options::variables_map& vm, bool testnet); + bool init(const boost::program_options::variables_map& vm); bool deinit(); bool send_stop_signal(); uint32_t get_this_peer_port(){return m_listenning_port;} @@ -113,6 +114,7 @@ namespace nodetool virtual uint64_t get_connections_count(); size_t get_outgoing_connections_count(); peerlist_manager& get_peerlist_manager(){return m_peerlist;} + void delete_connections(size_t count); private: const std::vector<std::string> m_seed_nodes_list = { "seeds.moneroseeds.se" @@ -120,6 +122,9 @@ namespace nodetool , "seeds.moneroseeds.ch" , "seeds.moneroseeds.li" }; + + bool islimitup=false; + bool islimitdown=false; typedef COMMAND_REQUEST_STAT_INFO_T<typename t_payload_net_handler::stat_info> COMMAND_REQUEST_STAT_INFO; @@ -199,6 +204,20 @@ namespace nodetool template <class Container> bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container); + bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max); + bool set_tos_flag(const boost::program_options::variables_map& vm, int limit); + + bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit); + bool set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit); + bool set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit); + + void kill() { ///< will be called e.g. from deinit() + _info("Killing the net_node"); + is_closing = true; + mPeersLoggerThread->join(); // make sure the thread finishes + _info("Joined extra background net_node threads"); + } + //debug functions std::string print_connections_container(); @@ -216,7 +235,16 @@ namespace nodetool END_KV_SERIALIZE_MAP() }; - config m_config; + public: + config m_config; // TODO was private, add getters? + std::atomic<unsigned int> m_current_number_of_out_peers; + + void set_save_graph(bool save_graph) + { + m_save_graph = save_graph; + epee::net_utils::connection_basic::set_save_graph(save_graph); + } + private: std::string m_config_folder; bool m_have_address; @@ -226,7 +254,10 @@ namespace nodetool uint32_t m_ip_address; bool m_allow_local_ip; bool m_hide_my_port; - + bool m_no_igd; + std::atomic<bool> m_save_graph; + std::atomic<bool> is_closing; + std::unique_ptr<std::thread> mPeersLoggerThread; //critical_section m_connections_lock; //connections_indexed_container m_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index b6337e6b5..cede6d0a0 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -31,6 +31,9 @@ #pragma once #include <algorithm> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include <atomic> #include "version.h" #include "string_tools.h" @@ -43,16 +46,18 @@ #include "net/local_ip.h" #include "crypto/crypto.h" #include "storages/levin_abstract_invoke2.h" +#include "data_logger.hpp" +#include "daemon/command_line_args.h" // We have to look for miniupnpc headers in different places, dependent on if its compiled or external #ifdef UPNP_STATIC - #include <miniupnpc/miniupnpc.h> - #include <miniupnpc/upnpcommands.h> - #include <miniupnpc/upnperrors.h> + #include <miniupnpc/miniupnpc.h> + #include <miniupnpc/upnpcommands.h> + #include <miniupnpc/upnperrors.h> #else - #include "miniupnpc.h" - #include "upnpcommands.h" - #include "upnperrors.h" + #include "miniupnpc.h" + #include "upnpcommands.h" + #include "upnperrors.h" #endif #define NET_MAKE_IP(b1,b2,b3,b4) ((LPARAM)(((DWORD)(b1)<<24)+((DWORD)(b2)<<16)+((DWORD)(b3)<<8)+((DWORD)(b4)))) @@ -81,6 +86,16 @@ namespace nodetool " If this option is given the options add-priority-node and seed-node are ignored"}; const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; const command_line::arg_descriptor<bool> arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; + + const command_line::arg_descriptor<bool> arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; + const command_line::arg_descriptor<int64_t> arg_out_peers = {"out-peers", "set max limit of out peers", -1}; + const command_line::arg_descriptor<int> arg_tos_flag = {"tos-flag", "set TOS flag", -1}; + + const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; + const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1}; + const command_line::arg_descriptor<uint64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", 128}; + + const command_line::arg_descriptor<bool> arg_save_graph = {"save-graph", "Save data for dr monero", false}; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -96,6 +111,13 @@ namespace nodetool command_line::add_arg(desc, arg_p2p_add_exclusive_node); command_line::add_arg(desc, arg_p2p_seed_node); command_line::add_arg(desc, arg_p2p_hide_my_port); + command_line::add_arg(desc, arg_no_igd); + command_line::add_arg(desc, arg_out_peers); + command_line::add_arg(desc, arg_tos_flag); + command_line::add_arg(desc, arg_limit_rate_up); + command_line::add_arg(desc, arg_limit_rate_down); + command_line::add_arg(desc, arg_limit_rate); + command_line::add_arg(desc, arg_save_graph); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -117,7 +139,6 @@ namespace nodetool //at this moment we have hardcoded config m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL; - m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; //20 MB limit m_config.m_net_config.config_id = 0; // initial config m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT; @@ -162,6 +183,7 @@ namespace nodetool m_port = command_line::get_arg(vm, p2p_bind_arg); m_external_port = command_line::get_arg(vm, arg_p2p_external_port); m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip); + m_no_igd = command_line::get_arg(vm, arg_no_igd); if (command_line::has_arg(vm, arg_p2p_add_peer)) { @@ -175,17 +197,24 @@ namespace nodetool m_command_line_peers.push_back(pe); } } + + if(command_line::has_arg(vm, arg_save_graph)) + { + set_save_graph(true); + } if (command_line::has_arg(vm,arg_p2p_add_exclusive_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_add_exclusive_node, m_exclusive_peers)) return false; } + if (command_line::has_arg(vm, arg_p2p_add_priority_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_add_priority_node, m_priority_peers)) return false; } + if (command_line::has_arg(vm, arg_p2p_seed_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_seed_node, m_seed_nodes)) @@ -194,6 +223,21 @@ namespace nodetool if(command_line::has_arg(vm, arg_p2p_hide_my_port)) m_hide_my_port = true; + + if ( !set_max_out_peers(vm, command_line::get_arg(vm, arg_out_peers) ) ) + return false; + + if ( !set_tos_flag(vm, command_line::get_arg(vm, arg_tos_flag) ) ) + return false; + + if ( !set_rate_up_limit(vm, command_line::get_arg(vm, arg_limit_rate_up) ) ) + return false; + + if ( !set_rate_down_limit(vm, command_line::get_arg(vm, arg_limit_rate_down) ) ) + return false; + + if ( !set_rate_limit(vm, command_line::get_arg(vm, arg_limit_rate) ) ) + return false; return true; } @@ -238,44 +282,117 @@ namespace nodetool //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm, bool testnet) + bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm) { + bool testnet = command_line::get_arg(vm, daemon_args::arg_testnet_on); + if (testnet) { - append_net_address(m_seed_nodes, "107.152.187.202:28080"); + memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16); append_net_address(m_seed_nodes, "197.242.158.240:28080"); append_net_address(m_seed_nodes, "107.152.130.98:28080"); + append_net_address(m_seed_nodes, "5.9.25.103:28080"); + append_net_address(m_seed_nodes, "5.9.55.70:28080"); } else { + memcpy(&m_network_id, &::config::NETWORK_ID, 16); // for each hostname in the seed nodes list, attempt to DNS resolve and // add the result addresses as seed nodes // TODO: at some point add IPv6 support, but that won't be relevant // for some time yet. + + std::vector<std::vector<std::string>> dns_results; + dns_results.resize(m_seed_nodes_list.size()); + + std::list<boost::thread*> dns_threads; + uint64_t result_index = 0; for (const std::string& addr_str : m_seed_nodes_list) { - // TODO: care about dnssec avail/valid - bool avail, valid; - std::vector<std::string> addr_list = tools::DNSResolver::instance().get_ipv4(addr_str, avail, valid); - for (const std::string& a : addr_list) + boost::thread* th = new boost::thread([=, &dns_results, &addr_str] + { + LOG_PRINT_L4("dns_threads[" << result_index << "] created for: " << addr_str) + // TODO: care about dnssec avail/valid + bool avail, valid; + std::vector<std::string> addr_list; + + try + { + addr_list = tools::DNSResolver().get_ipv4(addr_str, avail, valid); + LOG_PRINT_L4("dns_threads[" << result_index << "] DNS resolve done"); + boost::this_thread::interruption_point(); + } + catch(const boost::thread_interrupted&) + { + // thread interruption request + // even if we now have results, finish thread without setting + // result variables, which are now out of scope in main thread + LOG_PRINT_L4("dns_threads[" << result_index << "] interrupted"); + return; + } + + LOG_PRINT_L4("dns_threads[" << result_index << "] addr_str: " << addr_str << " number of results: " << addr_list.size()); + dns_results[result_index] = addr_list; + }); + + dns_threads.push_back(th); + ++result_index; + } + + LOG_PRINT_L4("dns_threads created, now waiting for completion or timeout of " << CRYPTONOTE_DNS_TIMEOUT_MS << "ms"); + boost::chrono::system_clock::time_point deadline = boost::chrono::system_clock::now() + boost::chrono::milliseconds(CRYPTONOTE_DNS_TIMEOUT_MS); + uint64_t i = 0; + for (boost::thread* th : dns_threads) + { + if (! th->try_join_until(deadline)) + { + LOG_PRINT_L4("dns_threads[" << i << "] timed out, sending interrupt"); + th->interrupt(); + } + ++i; + } + + i = 0; + for (const auto& result : dns_results) + { + LOG_PRINT_L4("DNS lookup for " << m_seed_nodes_list[i] << ": " << result.size() << " results"); + // if no results for node, thread's lookup likely timed out + if (result.size()) { - append_net_address(m_seed_nodes, a + ":18080"); + for (const auto& addr_string : result) + { + append_net_address(m_seed_nodes, addr_string + ":18080"); + } } + ++i; } if (!m_seed_nodes.size()) { - append_net_address(m_seed_nodes, "62.210.78.186:18080"); - append_net_address(m_seed_nodes, "195.12.60.154:18080"); - append_net_address(m_seed_nodes, "54.241.246.125:18080"); - append_net_address(m_seed_nodes, "107.170.157.169:18080"); - append_net_address(m_seed_nodes, "54.207.112.216:18080"); - append_net_address(m_seed_nodes, "78.27.112.54:18080"); - append_net_address(m_seed_nodes, "209.222.30.57:18080"); - append_net_address(m_seed_nodes, "80.71.13.55:18080"); - append_net_address(m_seed_nodes, "107.178.112.126:18080"); - append_net_address(m_seed_nodes, "107.158.233.98:18080"); - append_net_address(m_seed_nodes, "64.22.111.2:18080"); + LOG_PRINT_L0("DNS seed node lookup either timed out or failed, falling back to defaults"); + append_net_address(m_seed_nodes, "46.165.232.77:18080"); + append_net_address(m_seed_nodes, "63.141.254.186:18080"); + append_net_address(m_seed_nodes, ":18080"); + append_net_address(m_seed_nodes, "119.81.118.164:18080"); + append_net_address(m_seed_nodes, "60.191.33.112:18080"); + append_net_address(m_seed_nodes, "198.74.231.92:18080"); + append_net_address(m_seed_nodes, "5.9.55.70:18080"); + append_net_address(m_seed_nodes, "119.81.118.165:18080"); + append_net_address(m_seed_nodes, "202.112.0.100:18080"); + append_net_address(m_seed_nodes, "84.106.163.174:18080"); + append_net_address(m_seed_nodes, "178.206.94.87:18080"); + append_net_address(m_seed_nodes, "119.81.118.163:18080"); + append_net_address(m_seed_nodes, "95.37.217.253:18080"); + append_net_address(m_seed_nodes, "161.67.132.39:18080"); + append_net_address(m_seed_nodes, "119.81.48.114:18080"); + append_net_address(m_seed_nodes, "119.81.118.166:18080"); + append_net_address(m_seed_nodes, "93.120.240.209:18080"); + append_net_address(m_seed_nodes, "46.183.145.69:18080"); + append_net_address(m_seed_nodes, "108.170.123.66:18080"); + append_net_address(m_seed_nodes, "5.9.83.204:18080"); + append_net_address(m_seed_nodes, "104.130.19.193:18080"); + append_net_address(m_seed_nodes, "119.81.48.115:18080"); + append_net_address(m_seed_nodes, "80.71.13.36:18080"); } } @@ -316,42 +433,43 @@ namespace nodetool LOG_PRINT_L0("External port defined as " << m_external_port); // Add UPnP port mapping - LOG_PRINT_L0("Attempting to add IGD port mapping."); - int result; - UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); - UPNPUrls urls; - IGDdatas igdData; - char lanAddress[64]; - result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); - freeUPNPDevlist(deviceList); - if (result != 0) { - if (result == 1) { - std::ostringstream portString; - portString << m_listenning_port; - - // Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly - UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0); - - int portMappingResult; - portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0"); - if (portMappingResult != 0) { - LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult)); - } else { - LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0); - } - } else if (result == 2) { - LOG_PRINT_L0("IGD was found but reported as not connected."); - } else if (result == 3) { - LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD."); - } else { - LOG_ERROR("UPNP_GetValidIGD returned an unknown result code."); - } - - FreeUPNPUrls(&urls); - } else { - LOG_PRINT_L0("No IGD was found."); - } - + if(m_no_igd == false) { + LOG_PRINT_L0("Attempting to add IGD port mapping."); + int result; + UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); + UPNPUrls urls; + IGDdatas igdData; + char lanAddress[64]; + result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); + freeUPNPDevlist(deviceList); + if (result != 0) { + if (result == 1) { + std::ostringstream portString; + portString << m_listenning_port; + + // Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly + UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0); + + int portMappingResult; + portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0"); + if (portMappingResult != 0) { + LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult)); + } else { + LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0); + } + } else if (result == 2) { + LOG_PRINT_L0("IGD was found but reported as not connected."); + } else if (result == 3) { + LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD."); + } else { + LOG_ERROR("UPNP_GetValidIGD returned an unknown result code."); + } + + FreeUPNPUrls(&urls); + } else { + LOG_PRINT_L0("No IGD was found."); + } + } return res; } //----------------------------------------------------------------------------------- @@ -364,6 +482,30 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::run() { + // creating thread to log number of connections + mPeersLoggerThread.reset(new std::thread([&]() + { + _note("Thread monitor number of peers - start"); + while (!is_closing) + { // main loop of thread + //number_of_peers = m_net_server.get_config_object().get_connections_count(); + unsigned int number_of_peers = 0; + m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (!cntxt.m_is_income) ++number_of_peers; + return true; + }); // lambda + + m_current_number_of_out_peers = number_of_peers; + if (epee::net_utils::data_logger::is_dying()) + break; + epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + } // main loop of thread + _note("Thread monitor number of peers - done"); + })); // lambda + //here you can set worker threads count int thrds_count = 10; @@ -394,6 +536,7 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::deinit() { + kill(); m_peerlist.deinit(); m_net_server.deinit_server(); return store_config(); @@ -606,6 +749,16 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, bool white) { + if (m_current_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit + { + return false; + } + else if (m_current_number_of_out_peers > m_config.m_net_config.connections_count) + { + m_net_server.get_config_object().del_out_connections(1); + m_current_number_of_out_peers --; // atomic variable, update time = 1s + return false; + } LOG_PRINT_L1("Connecting to " << epee::string_tools::get_ip_string_from_int32(na.ip) << ":" << epee::string_tools::num_to_string_fast(na.port) << "(white=" << white << ", last_seen: " << (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never") @@ -693,16 +846,22 @@ namespace nodetool ++try_count; - if(is_peer_used(pe)) + _note("Considering connecting (out) to peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << boost::lexical_cast<std::string>(pe.adr.port)); + + if(is_peer_used(pe)) { + _note("Peer is used"); continue; + } LOG_PRINT_L1("Selected peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << boost::lexical_cast<std::string>(pe.adr.port) << "[white=" << use_white_list << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); - if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list)) + if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list)) { + _note("Handshake failed"); continue; + } return true; } @@ -1241,4 +1400,83 @@ namespace nodetool return true; } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) + { + if(max == -1) { + m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; + epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_config.m_net_config.connections_count); + return true; + } + epee::net_utils::data_logger::get_instance().add_data("peers_limit", max); + m_config.m_net_config.connections_count = max; + return true; + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::delete_connections(size_t count) + { + m_net_server.get_config_object().del_out_connections(count); + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_tos_flag(const boost::program_options::variables_map& vm, int flag) + { + if(flag==-1){ + return true; + } + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_tos_flag(flag); + _dbg1("Set ToS flag " << flag); + return true; + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit) + { + this->islimitup=true; + + if (limit==-1) { + limit=128; + this->islimitup=false; + } + + limit *= 1024; + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit ); + LOG_PRINT_L0("Set limit-up to " << limit/1024 << " kB/s"); + return true; + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit) + { + this->islimitdown=true; + if(limit==-1) { + limit=128; + this->islimitdown=false; + } + limit *= 1024; + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit ); + LOG_PRINT_L0("Set limit-down to " << limit/1024 << " kB/s"); + return true; + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit) + { + limit *= 1024; + if(this->islimitdown==false && this->islimitup==false) { + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit ); + LOG_PRINT_L0("Set limit to " << limit/1024 << " kB/s"); + } + else if(this->islimitdown==false && this->islimitup==true ) { + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit ); + } + else if(this->islimitdown==true && this->islimitup==false ) { + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit ); + } + + return true; + } } diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 6559963e0..a7b8bf6f3 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index e3e387bbc..9407e7bce 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 17525e8fe..d0bd44ef0 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp new file mode 100644 index 000000000..6fa27b62a --- /dev/null +++ b/src/p2p/network_throttle-detail.cpp @@ -0,0 +1,382 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for throttle details */ + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + + + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> +#include "../../contrib/epee/include/net/abstract_tcp_server2.h" + +// TODO: +#include "../../src/p2p/network_throttle-detail.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +#include "data_logger.hpp" +using namespace nOT::nUtils; + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +using namespace nOT::nUtils; + +namespace epee +{ +namespace net_utils +{ + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + void _packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) could be used for different kinds of sleep e.g. direct/queue write +}; + + +} // namespace +} // namespace + + + + + + +// ################################################################################################ +// ################################################################################################ +// The implementation part +// ################################################################################################ +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle +// ================================================================================================ + +network_throttle::~network_throttle() { } + +network_throttle::packet_info::packet_info() + : m_size(0) +{ +} + +network_throttle::network_throttle(const std::string &nameshort, const std::string &name, int window_size) + : m_window_size( (window_size==-1) ? 10 : window_size ), + m_history( m_window_size ), m_nameshort(nameshort) +{ + set_name(name); + m_network_add_cost = 128; + m_network_minimal_segment = 256; + m_network_max_segment = 1024*1024; + m_any_packet_yet = false; + m_slot_size = 1.0; // hard coded in few places + m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle +} + +void network_throttle::set_name(const std::string &name) +{ + m_name = name; +} + +void network_throttle::set_target_speed( network_speed_kbps target ) +{ + m_target_speed = target * 1024; + _note_c("net/"+m_nameshort, "Setting LIMIT: " << target << " kbps"); + set_real_target_speed(target); +} + +void network_throttle::set_real_target_speed( network_speed_kbps real_target ) +{ + m_real_target_speed = real_target * 1024; +} + +network_speed_kbps network_throttle::get_terget_speed() +{ + return m_real_target_speed / 1024; +} + +void network_throttle::tick() +{ + double time_now = get_time_seconds(); + if (!m_any_packet_yet) m_start_time = time_now; // starting now + + network_time_seconds current_sample_time_slot = time_to_slot( time_now ); // T=13.7 --> 13 (for 1-second smallwindow) + network_time_seconds last_sample_time_slot = time_to_slot( m_last_sample_time ); + + // moving to next position, and filling gaps + // !! during this loop the m_last_sample_time and last_sample_time_slot mean the variable moved in +1 + // TODO optimize when moving few slots at once + while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot)) + { + _dbg3("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")"); + // rotate buffer + for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1]; + m_history[0] = packet_info(); + if (! m_any_packet_yet) + { + m_last_sample_time = time_now; + } + m_last_sample_time += 1; last_sample_time_slot = time_to_slot( m_last_sample_time ); // increase and recalculate time, time slot + m_any_packet_yet=true; + } + m_last_sample_time = time_now; // the real exact last time +} + +void network_throttle::handle_trafic_exact(size_t packet_size) +{ + _handle_trafic_exact(packet_size, packet_size); +} + +void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_size) +{ + tick(); + + calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1); + calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5); + m_history[0].m_size += packet_size; + + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + + _dbg2_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)" + << " Speed AVG=" << std::setw(4) << ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]" + << " " << std::setw(4) << ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]" + <<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec " + << " " << history_str + ); +} + +void network_throttle::handle_trafic_tcp(size_t packet_size) +{ + size_t all_size = packet_size + m_network_add_cost; + all_size = std::max( m_network_minimal_segment , all_size); + _handle_trafic_exact( all_size , packet_size ); +} + +network_time_seconds network_throttle::get_sleep_time_after_tick(size_t packet_size) { + tick(); + return get_sleep_time(packet_size); +} + +void network_throttle::logger_handle_net(const std::string &filename, double time, size_t size) { + if (! epee::net_utils::data_logger::m_save_graph) + return; + std::mutex mutex; + mutex.lock(); { + std::fstream file; + file.open(filename.c_str(), std::ios::app | std::ios::out ); + file.precision(6); + if(!file.is_open()) + _warn("Can't open file " << filename); + file << static_cast<int>(time) << " " << static_cast<double>(size/1024) << "\n"; + file.close(); + } mutex.unlock(); +} + +// fine tune this to decide about sending speed: +network_time_seconds network_throttle::get_sleep_time(size_t packet_size) const +{ + double D2=0; + calculate_times_struct cts = { 0, 0, 0, 0}; + calculate_times(packet_size, cts, true, m_window_size); D2=cts.delay; + return D2; +} + +// MAIN LOGIC: +void network_throttle::calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const +{ + const double the_window_size = std::max( (double)m_window_size , + ((force_window>0) ? force_window : m_window_size) + ); + + if (!m_any_packet_yet) { + cts.window=0; cts.average=0; cts.delay=0; + cts.recomendetDataSize = m_network_minimal_segment; // should be overrided by caller anyway + return ; // no packet yet, I can not decide about sleep time + } + + network_time_seconds window_len = (the_window_size-1) * m_slot_size ; // -1 since current slot is not finished + window_len += (m_last_sample_time - time_to_slot(m_last_sample_time)); // add the time for current slot e.g. 13.7-13 = 0.7 + + auto time_passed = get_time_seconds() - m_start_time; + cts.window = std::max( std::min( window_len , time_passed ) , m_slot_size ) ; // window length resulting from size of history but limited by how long ago history was started, + // also at least slot size (e.g. 1 second) to not be ridiculous + // window_len e.g. 5.7 because takes into account current slot time + + size_t Epast = 0; // summ of traffic till now + for (auto sample : m_history) Epast += sample.m_size; + + const size_t E = Epast; + const size_t Enow = Epast + packet_size ; // including the data we're about to send now + + const double M = m_target_speed; // max + const double D1 = (Epast - M*cts.window) / M; // delay - how long to sleep to get back to target speed + const double D2 = (Enow - M*cts.window) / M; // delay - how long to sleep to get back to target speed (including current packet) + + cts.delay = (D1*0.80 + D2*0.20); // finall sleep depends on both with/without current packet + // update_overheat(); + cts.average = Epast/cts.window; // current avg. speed (for info) + + if (Epast <= 0) { + if (cts.delay>=0) cts.delay = 0; // no traffic in history so we will not wait + } + + double Wgood=-1; + { // how much data we recommend now to download + Wgood = the_window_size + 1; + cts.recomendetDataSize = M*cts.window - E; + } + + if (dbg) { + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + _dbg1_c( "net/"+m_nameshort+"_c" , + (cts.delay > 0 ? "SLEEP" : "") + << "dbg " << m_name << ": " + << "speed is A=" << std::setw(8) <<cts.average<<" vs " + << "Max=" << std::setw(8) <<M<<" " + << " so sleep: " + << "D=" << std::setw(8) <<cts.delay<<" sec " + << "E="<< std::setw(8) << E << " (Enow="<<std::setw(8)<<Enow<<") " + << "M=" << std::setw(8) << M <<" W="<< std::setw(8) << cts.window << " " + << "R=" << std::setw(8) << cts.recomendetDataSize << " Wgood" << std::setw(8) << Wgood << " " + << "History: " << std::setw(8) << history_str << " " + << "m_last_sample_time=" << std::setw(8) << m_last_sample_time + ); + + } +} + +double network_throttle::get_time_seconds() const { + #if defined(__APPLE__) + auto point = std::chrono::system_clock::now(); + #else + auto point = std::chrono::steady_clock::now(); + #endif + auto time_from_epoh = point.time_since_epoch(); + auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + return ms_f / 1000.; +} + +size_t network_throttle::get_recommended_size_of_planned_transport_window(double force_window) const { + calculate_times_struct cts = { 0, 0, 0, 0}; + network_throttle::calculate_times(0, cts, true, force_window); + cts.recomendetDataSize += m_network_add_cost; + if (cts.recomendetDataSize<0) cts.recomendetDataSize=0; + if (cts.recomendetDataSize>m_network_max_segment) cts.recomendetDataSize=m_network_max_segment; + size_t RI = (long int)cts.recomendetDataSize; + return RI; +} + +size_t network_throttle::get_recommended_size_of_planned_transport() const { + size_t R1=0,R2=0,R3=0; + R1 = get_recommended_size_of_planned_transport_window( -1 ); + R2 = get_recommended_size_of_planned_transport_window(m_window_size / 2); + R3 = get_recommended_size_of_planned_transport_window( 5 ); + auto RM = std::min(R1, std::min(R2,R3)); + + const double a1=20, a2=10, a3=10, am=10; // weight of the various windows in decisssion // TODO 70 => 20 + return (R1*a1 + R2*a2 + R3*a3 + RM*am) / (a1+a2+a3+am); +} + +double network_throttle::get_current_speed() const { + unsigned int bytes_transferred = 0; + if (m_history.size() == 0 || m_slot_size == 0) + return 0; + + auto it = m_history.begin(); + while (it < m_history.end() - 1) + { + bytes_transferred += it->m_size; + it ++; + } + + return bytes_transferred / ((m_history.size() - 1) * m_slot_size); +} + +} // namespace +} // namespace + diff --git a/src/p2p/network_throttle-detail.hpp b/src/p2p/network_throttle-detail.hpp new file mode 100644 index 000000000..063dac850 --- /dev/null +++ b/src/p2p/network_throttle-detail.hpp @@ -0,0 +1,131 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: throttle details, implementing rate limiting */ + + +#ifndef INCLUDED_src_p2p_throttle_detail_hpp +#define INCLUDED_src_p2p_throttle_detail_hpp + +#include "../../src/p2p/network_throttle.hpp" + +namespace epee +{ +namespace net_utils +{ + + +class network_throttle : public i_network_throttle { + private: + struct packet_info { + size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second) + packet_info(); + }; + + + network_speed_kbps m_target_speed; + network_speed_kbps m_real_target_speed; + size_t m_network_add_cost; // estimated add cost of headers + size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to + size_t m_network_max_segment; // recommended max size of 1 TCP transmission + + const size_t m_window_size; // the number of samples to average over + network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot() + // TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements + + std::vector< packet_info > m_history; // the history of bw usage + network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer + network_time_seconds m_start_time; // when we were created + bool m_any_packet_yet; // did we yet got any packet to count + + double m_overheat; // last overheat + double m_overheat_time; // time in seconds after epoch + + std::string m_name; // my name for debug and logs + std::string m_nameshort; // my name for debug and logs (used in log file name) + + // each sample is now 1 second + public: + network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1); + virtual ~network_throttle(); + virtual void set_name(const std::string &name); + virtual void set_target_speed( network_speed_kbps target ); + virtual void set_real_target_speed( network_speed_kbps real_target ); // only for throttle_out + virtual network_speed_kbps get_terget_speed(); + + // add information about events: + virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc + + virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc) + + virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic + + // time calculations: + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info) + + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size + virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO + + virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported + virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame + virtual double get_current_speed() const; + + private: + virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 + virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size); + virtual void logger_handle_net(const std::string &filename, double time, size_t size); +}; + +/*** + * The complete set of traffic throttle for one typical connection +*/ +struct network_throttle_bw { + public: + network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually) + network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually) + network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually) + + public: + network_throttle_bw(const std::string &name1); +}; + + + +} // namespace net_utils +} // namespace epee + + +#endif + + diff --git a/src/p2p/network_throttle.cpp b/src/p2p/network_throttle.cpp new file mode 100644 index 000000000..7bc89881d --- /dev/null +++ b/src/p2p/network_throttle.cpp @@ -0,0 +1,120 @@ +/** +@file +@author rfree (current maintainer in monero.cc project) +@brief interface for throttling of connection (count and rate-limit speed etc) +@details <PRE> + +Throttling work by: +1) taking note of all traffic (hooks added e.g. to connection class) and measuring speed +2) depending on that information we sleep before sending out data (or send smaller portions of data) +3) depending on the information we can also sleep before sending requests or ask for smaller sets of data to download + +</PRE> + +@image html images/net/rate1-down-1k.png +@image html images/net/rate1-down-full.png +@image html images/net/rate1-up-10k.png +@image html images/net/rate1-up-full.png +@image html images/net/rate2-down-100k.png +@image html images/net/rate2-down-10k.png +@image html images/net/rate2-down-50k.png +@image html images/net/rate2-down-full.png +@image html images/net/rate2-up-100k.png +@image html images/net/rate2-up-10k.png +@image html images/net/rate3-up-10k.png + + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "../../src/p2p/network_throttle-detail.hpp" + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle_manager +// ================================================================================================ + +// ================================================================================================ +// static: +std::mutex network_throttle_manager::m_lock_get_global_throttle_in; +std::mutex network_throttle_manager::m_lock_get_global_throttle_inreq; +std::mutex network_throttle_manager::m_lock_get_global_throttle_out; + +int network_throttle_manager::xxx; + + +// ================================================================================================ +// methods: +i_network_throttle & network_throttle_manager::get_global_throttle_in() { + std::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } ); + return * m_obj_get_global_throttle_in; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_in; +std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_in; + + + +i_network_throttle & network_throttle_manager::get_global_throttle_inreq() { + std::call_once(m_once_get_global_throttle_inreq, [] { m_obj_get_global_throttle_inreq.reset(new network_throttle("inreq/all", "<== global-IN-REQ",10)); } ); + return * m_obj_get_global_throttle_inreq; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_inreq; +std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_inreq; + + +i_network_throttle & network_throttle_manager::get_global_throttle_out() { + std::call_once(m_once_get_global_throttle_out, [] { m_obj_get_global_throttle_out.reset(new network_throttle("out/all", ">>> global-OUT",10)); } ); + return * m_obj_get_global_throttle_out; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_out; +std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_out; + + + + +network_throttle_bw::network_throttle_bw(const std::string &name1) + : m_in("in/"+name1, name1+"-DOWNLOAD"), m_inreq("inreq/"+name1, name1+"-DOWNLOAD-REQUESTS"), m_out("out/"+name1, name1+"-UPLOAD") +{ } + + + + +} // namespace +} // namespace + + + + + diff --git a/src/p2p/network_throttle.hpp b/src/p2p/network_throttle.hpp new file mode 100644 index 000000000..add4daa86 --- /dev/null +++ b/src/p2p/network_throttle.hpp @@ -0,0 +1,185 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief interface for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: throttle basic interface */ +/* rfree: also includes the manager for singeton/global such objects */ + + +#ifndef INCLUDED_p2p_network_throttle_hpp +#define INCLUDED_p2p_network_throttle_hpp + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + +#include <memory> +#include <mutex> +#include <fstream> + +namespace epee +{ +namespace net_utils +{ + +// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono? +typedef double network_speed_kbps; +typedef double network_time_seconds; +typedef double network_MB; + +class i_network_throttle; + +/*** +@brief All information about given throttle - speed calculations +*/ +struct calculate_times_struct { + double average; + double window; + double delay; + double recomendetDataSize; +}; +typedef calculate_times_struct calculate_times_struct; + + +namespace cryptonote { class cryptonote_protocol_handler_base; } // a friend class // TODO friend not working + +/*** +@brief Access to simple throttles, with singlton to access global network limits +*/ +class network_throttle_manager { + // provides global (singleton) in/inreq/out throttle access + + // [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/ + // [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests) + + //protected: + public: // XXX + // [[note1]] + static std::once_flag m_once_get_global_throttle_in; + static std::once_flag m_once_get_global_throttle_inreq; // [[note2]] + static std::once_flag m_once_get_global_throttle_out; + static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_in; + static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_inreq; + static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_out; + + static std::mutex m_lock_get_global_throttle_in; + static std::mutex m_lock_get_global_throttle_inreq; + static std::mutex m_lock_get_global_throttle_out; + + friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic_pimpl; // ditto + + static int xxx; + + public: + static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in + static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously + static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously +}; + + + +/*** +@brief interface for the throttle, see the derivated class +*/ +class i_network_throttle { + public: + virtual void set_name(const std::string &name)=0; + virtual void set_target_speed( network_speed_kbps target )=0; + virtual void set_real_target_speed(network_speed_kbps real_target)=0; + virtual network_speed_kbps get_terget_speed()=0; + + virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc + virtual void tick() =0; // poke and update timers/history + + // time calculations: + + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate: + // Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size + + // Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now + + virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer + + virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future + + virtual double get_time_seconds() const =0; // a timer + virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + + +}; + + +// ... more in the -advanced.h file + + +} // namespace net_utils +} // namespace epee + + +#endif + + + diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index 44b8c5b50..e157f3188 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/stdafx.h b/src/p2p/stdafx.h index ede53d517..da1216924 100644 --- a/src/p2p/stdafx.h +++ b/src/p2p/stdafx.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/mingw/alloca.h b/src/platform/mingw/alloca.h index 32589db06..868db1af4 100644 --- a/src/platform/mingw/alloca.h +++ b/src/platform/mingw/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/alloca.h b/src/platform/msc/alloca.h index 82186553b..c212d64b9 100644 --- a/src/platform/msc/alloca.h +++ b/src/platform/msc/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/inline_c.h b/src/platform/msc/inline_c.h index a016b783e..fa4fd48f3 100644 --- a/src/platform/msc/inline_c.h +++ b/src/platform/msc/inline_c.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/stdbool.h b/src/platform/msc/stdbool.h index 1044d2539..e3c932ccf 100644 --- a/src/platform/msc/stdbool.h +++ b/src/platform/msc/stdbool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/sys/param.h b/src/platform/msc/sys/param.h index 07a036a6e..0db360ab0 100644 --- a/src/platform/msc/sys/param.h +++ b/src/platform/msc/sys/param.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 5417a0ec1..cb8a8426c 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -45,6 +45,7 @@ bitmonero_add_library(rpc target_link_libraries(rpc LINK_PRIVATE cryptonote_core + cryptonote_protocol ${Boost_CHRONO_LIBRARY} ${Boost_REGEX_LIBRARY} ${Boost_SYSTEM_LIBRARY} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 036cb64ff..561161950 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -40,29 +40,10 @@ using namespace epee; #include "misc_language.h" #include "crypto/hash.h" #include "core_rpc_server_error_codes.h" +#include "daemon/command_line_args.h" namespace cryptonote { - namespace - { - const command_line::arg_descriptor<std::string> arg_rpc_bind_ip = { - "rpc-bind-ip" - , "IP for RPC server" - , "127.0.0.1" - }; - - const command_line::arg_descriptor<std::string> arg_rpc_bind_port = { - "rpc-bind-port" - , "Port for RPC server" - , std::to_string(config::RPC_DEFAULT_PORT) - }; - - const command_line::arg_descriptor<std::string> arg_testnet_rpc_bind_port = { - "testnet-rpc-bind-port" - , "Port for testnet RPC server" - , std::to_string(config::testnet::RPC_DEFAULT_PORT) - }; - } //----------------------------------------------------------------------------------- void core_rpc_server::init_options(boost::program_options::options_description& desc) @@ -75,11 +56,9 @@ namespace cryptonote core_rpc_server::core_rpc_server( core& cr , nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& p2p - , bool testnet ) : m_core(cr) , m_p2p(p2p) - , m_testnet(testnet) {} //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::handle_command_line( @@ -97,6 +76,8 @@ namespace cryptonote const boost::program_options::variables_map& vm ) { + m_testnet = command_line::get_arg(vm, daemon_args::arg_testnet_on); + m_net_server.set_threads_prefix("RPC"); bool r = handle_command_line(vm); CHECK_AND_ASSERT_MES(r, false, "Failed to process command line in core_rpc_server"); @@ -124,7 +105,7 @@ namespace cryptonote #define CHECK_CORE_READY() do { if(!check_core_ready()){res.status = CORE_RPC_STATUS_BUSY;return true;} } while(0) //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx) + bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res) { CHECK_CORE_BUSY(); res.height = m_core.get_current_blockchain_height(); @@ -132,7 +113,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, connection_context& cntx) + bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res) { CHECK_CORE_BUSY(); res.height = m_core.get_current_blockchain_height(); @@ -150,7 +131,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx) + bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) { CHECK_CORE_BUSY(); std::list<std::pair<block, std::list<transaction> > > bs; @@ -175,7 +156,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res, connection_context& cntx) + bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) { CHECK_CORE_BUSY(); res.status = "Failed"; @@ -204,7 +185,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, connection_context& cntx) + bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) { CHECK_CORE_BUSY(); bool r = m_core.get_tx_outputs_gindexs(req.txid, res.o_indexes); @@ -218,7 +199,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, connection_context& cntx) + bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res) { CHECK_CORE_BUSY(); std::vector<crypto::hash> vh; @@ -260,7 +241,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, connection_context& cntx) + bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res) { CHECK_CORE_READY(); @@ -303,7 +284,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, connection_context& cntx) + bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res) { CHECK_CORE_READY(); account_public_address adr; @@ -325,7 +306,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, connection_context& cntx) + bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res) { if(!m_core.get_miner().stop()) { @@ -336,7 +317,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, connection_context& cntx) + bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res) { CHECK_CORE_READY(); @@ -354,7 +335,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, connection_context& cntx) + bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res) { CHECK_CORE_BUSY(); if( !m_core.get_blockchain_storage().store_blockchain() ) @@ -366,7 +347,76 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, connection_context& cntx) + bool core_rpc_server::on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res) + { + /* + std::list<nodetool::peerlist_entry> white_list; + std::list<nodetool::peerlist_entry> gray_list; + m_p2p.get_peerlist(white_list, gray_list); + + for (auto & entry : white_list) + { + res.white_list.emplace_back(entry.id, entry.adr.ip, entry.adr.port, entry.last_seen); + } + + for (auto & entry : gray_list) + { + res.gray_list.emplace_back(entry.id, entry.adr.ip, entry.adr.port, entry.last_seen); + } + + */ + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res) + { + if(m_core.get_miner().is_mining()) + { + m_core.get_miner().do_print_hashrate(req.visible); + res.status = CORE_RPC_STATUS_OK; + } + else + { + res.status = CORE_RPC_STATUS_NOT_MINING; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res) + { + if (req.level < LOG_LEVEL_MIN || req.level > LOG_LEVEL_MAX) + { + res.status = "Error: log level not valid"; + } + else + { + epee::log_space::log_singletone::get_set_log_detalisation_level(true, req.level); + int otshell_utils_log_level = 100 - (req.level * 20); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); + res.status = CORE_RPC_STATUS_OK; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res) + { + CHECK_CORE_BUSY(); + m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res) + { + // FIXME: replace back to original m_p2p.send_stop_signal() after + // investigating why that isn't working quite right. + m_p2p.send_stop_signal(); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res) { CHECK_CORE_BUSY(); res.count = m_core.get_current_blockchain_height(); @@ -374,7 +424,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -415,7 +465,7 @@ namespace cryptonote return 0; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp) { if(!check_core_ready()) { @@ -475,12 +525,13 @@ namespace cryptonote LOG_ERROR("Failed to calculate offset for "); return false; } + res.prev_hash = string_tools::pod_to_hex(b.prev_id); res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp) { CHECK_CORE_READY(); if(req.size()!=1) @@ -552,7 +603,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -588,7 +639,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx){ + bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp){ if(!check_core_busy()) { error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; @@ -629,7 +680,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, connection_context& cntx){ + bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp){ if(!check_core_busy()) { error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; @@ -662,7 +713,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -678,7 +729,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -702,4 +753,62 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ -} + bool core_rpc_server::on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res) + { + cryptonote::core::set_fast_exit(); + m_p2p.deinit(); + m_core.deinit(); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res) + { + // TODO + /*if (m_p2p.get_outgoing_connections_count() > req.out_peers) + { + m_p2p.m_config.m_net_config.connections_count = req.out_peers; + if (m_p2p.get_outgoing_connections_count() > req.out_peers) + { + int count = m_p2p.get_outgoing_connections_count() - req.out_peers; + m_p2p.delete_connections(count); + } + } + + else + m_p2p.m_config.m_net_config.connections_count = req.out_peers; + */ + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res) + { + m_p2p.set_save_graph(true); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res) + { + m_p2p.set_save_graph(false); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_ip = { + "rpc-bind-ip" + , "IP for RPC server" + , "127.0.0.1" + }; + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_port = { + "rpc-bind-port" + , "Port for RPC server" + , std::to_string(config::RPC_DEFAULT_PORT) + }; + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_testnet_rpc_bind_port = { + "testnet-rpc-bind-port" + , "Port for testnet RPC server" + , std::to_string(config::testnet::RPC_DEFAULT_PORT) + }; + +} // namespace cryptonote diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 02bc533c9..cee8df25d 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -39,6 +39,10 @@ #include "p2p/net_node.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" +// yes, epee doesn't properly use its full namespace when calling its +// functions from macros. *sigh* +using namespace epee; + namespace cryptonote { /************************************************************************/ @@ -47,19 +51,22 @@ namespace cryptonote class core_rpc_server: public epee::http_server_impl_base<core_rpc_server> { public: + + static const command_line::arg_descriptor<std::string> arg_rpc_bind_ip; + static const command_line::arg_descriptor<std::string> arg_rpc_bind_port; + static const command_line::arg_descriptor<std::string> arg_testnet_rpc_bind_port; + typedef epee::net_utils::connection_context_base connection_context; core_rpc_server( core& cr , nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& p2p - , bool testnet ); static void init_options(boost::program_options::options_description& desc); bool init( const boost::program_options::variables_map& vm ); - private: CHAIN_HTTP_TO_MAP2(connection_context); //forward http requests to uri map @@ -74,7 +81,16 @@ namespace cryptonote MAP_URI_AUTO_JON2("/stop_mining", on_stop_mining, COMMAND_RPC_STOP_MINING) MAP_URI_AUTO_JON2("/mining_status", on_mining_status, COMMAND_RPC_MINING_STATUS) MAP_URI_AUTO_JON2("/save_bc", on_save_bc, COMMAND_RPC_SAVE_BC) + MAP_URI_AUTO_JON2("/get_peer_list", on_get_peer_list, COMMAND_RPC_GET_PEER_LIST) + MAP_URI_AUTO_JON2("/set_log_hash_rate", on_set_log_hash_rate, COMMAND_RPC_SET_LOG_HASH_RATE) + MAP_URI_AUTO_JON2("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL) + MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL) + MAP_URI_AUTO_JON2("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON) MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO) + MAP_URI_AUTO_JON2("/fast_exit", on_fast_exit, COMMAND_RPC_FAST_EXIT) + MAP_URI_AUTO_JON2("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS) + MAP_URI_AUTO_JON2("/start_save_graph", on_start_save_graph, COMMAND_RPC_START_SAVE_GRAPH) + MAP_URI_AUTO_JON2("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH) BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) @@ -88,29 +104,41 @@ namespace cryptonote END_JSON_RPC_MAP() END_URI_MAP2() - bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx); - bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx); - bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, connection_context& cntx); - bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, connection_context& cntx); - bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, connection_context& cntx); - bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, connection_context& cntx); - bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, connection_context& cntx); - bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, connection_context& cntx); - bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res, connection_context& cntx); - bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, connection_context& cntx); - bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, connection_context& cntx); + bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res); + bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res); + bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res); + bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res); + bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res); + bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); + bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); + bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); + bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); + bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); + bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); + bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); + bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res); + bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); + bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res); + bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); + bool on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res); + bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res); + bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); + bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); //json_rpc - bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, connection_context& cntx); - bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); + bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); + bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp); + bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp); + bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp); + bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp); + bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp); + bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp); + bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp); + bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp); //----------------------- + +private: + bool handle_command_line( const boost::program_options::variables_map& vm ); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index d711e1732..b6a2edd0b 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -39,6 +39,7 @@ namespace cryptonote //----------------------------------------------- #define CORE_RPC_STATUS_OK "OK" #define CORE_RPC_STATUS_BUSY "BUSY" +#define CORE_RPC_STATUS_NOT_MINING "NOT MINING" struct COMMAND_RPC_GET_HEIGHT { @@ -383,6 +384,7 @@ namespace cryptonote uint64_t difficulty; uint64_t height; uint64_t reserved_offset; + std::string prev_hash; blobdata blocktemplate_blob; std::string status; @@ -390,6 +392,7 @@ namespace cryptonote KV_SERIALIZE(difficulty) KV_SERIALIZE(height) KV_SERIALIZE(reserved_offset) + KV_SERIALIZE(prev_hash) KV_SERIALIZE(blocktemplate_blob) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() @@ -438,7 +441,7 @@ namespace cryptonote KV_SERIALIZE(reward) END_KV_SERIALIZE_MAP() }; - + struct COMMAND_RPC_GET_LAST_BLOCK_HEADER { struct request @@ -508,6 +511,148 @@ namespace cryptonote }; + struct peer { + uint64_t id; + uint32_t ip; + uint16_t port; + uint64_t last_seen; + + peer() = default; + + peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen) + : id(id), ip(ip), port(port), last_seen(last_seen) + {} + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(ip) + KV_SERIALIZE(port) + KV_SERIALIZE(last_seen) + END_KV_SERIALIZE_MAP() + }; + + struct COMMAND_RPC_GET_PEER_LIST + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<peer> white_list; + std::vector<peer> gray_list; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(white_list) + KV_SERIALIZE(gray_list) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SET_LOG_HASH_RATE + { + struct request + { + bool visible; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(visible) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SET_LOG_LEVEL + { + struct request + { + int8_t level; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(level) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct tx_info + { + std::string id_hash; + std::string tx_json; // TODO - expose this data directly + uint64_t blob_size; + uint64_t fee; + std::string max_used_block_id_hash; + uint64_t max_used_block_height; + bool kept_by_block; + uint64_t last_failed_height; + std::string last_failed_id_hash; + uint64_t receive_time; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id_hash) + KV_SERIALIZE(tx_json) + KV_SERIALIZE(blob_size) + KV_SERIALIZE(fee) + KV_SERIALIZE(max_used_block_id_hash) + KV_SERIALIZE(max_used_block_height) + KV_SERIALIZE(kept_by_block) + KV_SERIALIZE(last_failed_height) + KV_SERIALIZE(last_failed_id_hash) + KV_SERIALIZE(receive_time) + END_KV_SERIALIZE_MAP() + }; + + struct spent_key_image_info + { + std::string id_hash; + std::vector<std::string> txs_hashes; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id_hash) + KV_SERIALIZE(txs_hashes) + END_KV_SERIALIZE_MAP() + }; + + struct COMMAND_RPC_GET_TRANSACTION_POOL + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<tx_info> transactions; + std::vector<spent_key_image_info> spent_key_images; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(transactions) + KV_SERIALIZE(spent_key_images) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_GET_CONNECTIONS { struct request @@ -528,5 +673,122 @@ namespace cryptonote }; }; + + struct COMMAND_RPC_GET_BLOCK_HEADERS_RANGE + { + struct request + { + uint64_t start_height; + uint64_t end_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(start_height) + KV_SERIALIZE(end_height) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<block_header_responce> headers; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(headers) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_STOP_DAEMON + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_FAST_EXIT + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_OUT_PEERS + { + struct request + { + uint64_t out_peers; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(out_peers) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_START_SAVE_GRAPH + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_STOP_SAVE_GRAPH + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; } diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index ab6ca9aef..91659eb2a 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 56c94684e..5cd4988e4 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -42,8 +42,8 @@ #include "warnings.h" /* I have no clue what these lines means */ -PUSH_WARNINGS; -DISABLE_VS_WARNINGS(4244); +PUSH_WARNINGS +DISABLE_VS_WARNINGS(4244) //TODO: fix size_t warning in x32 platform diff --git a/src/serialization/binary_utils.h b/src/serialization/binary_utils.h index f2cfadd4e..4fe782326 100644 --- a/src/serialization/binary_utils.h +++ b/src/serialization/binary_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h index 8986d8081..f18e85b12 100644 --- a/src/serialization/crypto.h +++ b/src/serialization/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index 127e57ed6..4d697c14c 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index bd8b16130..49bbf95b6 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_utils.h b/src/serialization/json_utils.h index 64f9f946f..462d1f8e7 100644 --- a/src/serialization/json_utils.h +++ b/src/serialization/json_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 9e534cbc6..c94c99336 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/string.h b/src/serialization/string.h index 077f657d2..0f1b0d4e0 100644 --- a/src/serialization/string.h +++ b/src/serialization/string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/variant.h b/src/serialization/variant.h index 9b5d54237..9efa9c346 100644 --- a/src/serialization/variant.h +++ b/src/serialization/variant.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/vector.h b/src/serialization/vector.h index a099ce070..3787f00e6 100644 --- a/src/serialization/vector.h +++ b/src/serialization/vector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index 14f877907..a33ed0f32 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -50,6 +50,7 @@ target_link_libraries(simplewallet crypto common mnemonics + p2p ${UNBOUND_LIBRARY} ${UPNP_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} diff --git a/src/simplewallet/password_container.cpp b/src/simplewallet/password_container.cpp index 1c4aee453..e5fb933bb 100644 --- a/src/simplewallet/password_container.cpp +++ b/src/simplewallet/password_container.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/simplewallet/password_container.h b/src/simplewallet/password_container.h index b39fa1fe4..8a4191c7a 100644 --- a/src/simplewallet/password_container.h +++ b/src/simplewallet/password_container.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 4f7df2d3a..9ac80fa9e 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -67,6 +67,9 @@ namespace po = boost::program_options; #define EXTENDED_LOGS_FILE "wallet_details.log" +unsigned int epee::g_test_dbg_lock_sleep = 0; + +#define DEFAULT_MIX 3 namespace { @@ -81,6 +84,7 @@ namespace const command_line::arg_descriptor<int> arg_daemon_port = {"daemon-port", "Use daemon instance at port <arg> instead of 8081", 0}; const command_line::arg_descriptor<uint32_t> arg_log_level = {"set_log", "", 0, true}; const command_line::arg_descriptor<bool> arg_testnet = {"testnet", "Used to deploy test nets. The daemon must be launched with --testnet flag", false}; + const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts RPC to view only commands", false}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; @@ -231,6 +235,36 @@ bool simple_wallet::seed(const std::vector<std::string> &args/* = std::vector<st return true; } +bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + bool success = false; + if (!m_wallet->is_deterministic()) + { + fail_msg_writer() << "This wallet is non-deterministic and doesn't have a seed."; + return true; + } + tools::password_container pwd_container; + success = pwd_container.read_password(); + if (!success) + { + fail_msg_writer() << "failed to read wallet password"; + return true; + } + + /* verify password before using so user doesn't accidentally set a new password for rewritten wallet */ + success = m_wallet->verify_password(pwd_container.password()); + if (!success) + { + fail_msg_writer() << "invalid password"; + return true; + } + + std::string mnemonic_language = get_mnemonic_language(); + m_wallet->set_seed_language(mnemonic_language); + m_wallet->rewrite(m_wallet_file, pwd_container.password()); + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { success_msg_writer() << get_commands_str(); @@ -249,15 +283,45 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("incoming_transfers", boost::bind(&simple_wallet::show_incoming_transfers, this, _1), "incoming_transfers [available|unavailable] - Show incoming transfers - all of them or filter them by availability"); m_cmd_binder.set_handler("payments", boost::bind(&simple_wallet::show_payments, this, _1), "payments <payment_id_1> [<payment_id_2> ... <payment_id_N>] - Show payments <payment_id_1>, ... <payment_id_N>"); m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), "Show blockchain height"); - m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), "transfer <mixin_count> <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of transactions yours is indistinguishable from (from 0 to maximum available)"); + m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), "transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of transactions yours is indistinguishable from (from 0 to maximum available)"); m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), "set_log <level> - Change current log detalization level, <level> is a number 0-4"); m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), "Show current wallet public address"); m_cmd_binder.set_handler("save", boost::bind(&simple_wallet::save, this, _1), "Save wallet synchronized data"); m_cmd_binder.set_handler("viewkey", boost::bind(&simple_wallet::viewkey, this, _1), "Get viewkey"); m_cmd_binder.set_handler("seed", boost::bind(&simple_wallet::seed, this, _1), "Get deterministic seed"); + m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), "available options: seed language - Set wallet seed langage"); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), "Show this help"); } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::set_variable(const std::vector<std::string> &args) +{ + if (args.empty()) + { + fail_msg_writer() << "set: needs an argument. available options: seed"; + return true; + } + else + { + if (args[0] == "seed") + { + if (args.size() == 1) + { + fail_msg_writer() << "set seed: needs an argument. available options: language"; + return true; + } + else if (args[1] == "language") + { + std::vector<std::string> local_args = args; + local_args.erase(local_args.begin(), local_args.begin()+2); + seed_set_language(local_args); + return true; + } + } + } + fail_msg_writer() << "set: unrecognized argument(s)"; + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::set_log(const std::vector<std::string> &args) { if(args.size() != 1) @@ -285,14 +349,28 @@ bool simple_wallet::ask_wallet_create_if_needed() { std::string wallet_path; - wallet_path = command_line::input_line( - "Specify wallet file name (e.g., wallet.bin). If the wallet doesn't exist, it will be created.\n" - "Wallet file name: " - ); + bool valid_path = false; + do { + wallet_path = command_line::input_line( + "Specify wallet file name (e.g., wallet.bin). If the wallet doesn't exist, it will be created.\n" + "Wallet file name: " + ); + valid_path = tools::wallet2::wallet_valid_path_format(wallet_path); + if (!valid_path) + { + fail_msg_writer() << "wallet file path not valid: " << wallet_path; + } + } + while (!valid_path); bool keys_file_exists; bool wallet_file_exists; tools::wallet2::wallet_exists(wallet_path, keys_file_exists, wallet_file_exists); + LOG_PRINT_L3("wallet_path: " << wallet_path << ""); + LOG_PRINT_L3("keys_file_exists: " << std::boolalpha << keys_file_exists << std::noboolalpha + << " wallet_file_exists: " << std::boolalpha << wallet_file_exists << std::noboolalpha); + + LOG_PRINT_L1("Loading wallet..."); // add logic to error out if new wallet requested but named wallet file exists if (keys_file_exists || wallet_file_exists) @@ -575,6 +653,12 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string //---------------------------------------------------------------------------------------------------- bool simple_wallet::open_wallet(const string &wallet_file, const std::string& password, bool testnet) { + if (!tools::wallet2::wallet_valid_path_format(wallet_file)) + { + fail_msg_writer() << "wallet file path not valid: " << wallet_file; + return false; + } + m_wallet_file = wallet_file; m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); @@ -986,19 +1070,24 @@ bool simple_wallet::transfer(const std::vector<std::string> &args_) return true; std::vector<std::string> local_args = args_; - if(local_args.size() < 3) - { - fail_msg_writer() << "wrong number of arguments, expected at least 3, got " << local_args.size(); - return true; - } size_t fake_outs_count; - if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0])) + if(local_args.size() > 0) { + if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0])) + { + fake_outs_count = DEFAULT_MIX; + } + else + { + local_args.erase(local_args.begin()); + } + } + + if(local_args.size() < 2) { - fail_msg_writer() << "mixin_count should be non-negative integer, got " << local_args[0]; - return true; + fail_msg_writer() << "wrong number of arguments"; + return true; } - local_args.erase(local_args.begin()); std::vector<uint8_t> extra; if (1 == local_args.size() % 2) @@ -1256,6 +1345,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_non_deterministic ); command_line::add_arg(desc_params, arg_electrum_seed ); command_line::add_arg(desc_params, arg_testnet); + command_line::add_arg(desc_params, arg_restricted); tools::wallet_rpc_server::init_options(desc_params); po::positional_options_description positional_options; @@ -1326,6 +1416,7 @@ int main(int argc, char* argv[]) } bool testnet = command_line::get_arg(vm, arg_testnet); + bool restricted = command_line::get_arg(vm, arg_restricted); std::string wallet_file = command_line::get_arg(vm, arg_wallet_file); std::string wallet_password = command_line::get_arg(vm, arg_password); std::string daemon_address = command_line::get_arg(vm, arg_daemon_address); @@ -1338,7 +1429,7 @@ int main(int argc, char* argv[]) if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); - tools::wallet2 wal(testnet); + tools::wallet2 wal(testnet,restricted); try { LOG_PRINT_L0("Loading wallet..."); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index ebc85ed17..a8fe78414 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -81,6 +81,17 @@ namespace cryptonote bool viewkey(const std::vector<std::string> &args = std::vector<std::string>()); bool seed(const std::vector<std::string> &args = std::vector<std::string>()); + + /*! + * \brief Sets seed language. + * + * interactive + * - prompts for password so wallet can be rewritten + * - calls get_mnemonic_language() which prompts for language + * + * \return success status + */ + bool seed_set_language(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -96,6 +107,7 @@ namespace cryptonote ); bool print_address(const std::vector<std::string> &args = std::vector<std::string>()); bool save(const std::vector<std::string> &args); + bool set_variable(const std::vector<std::string> &args); bool set_log(const std::vector<std::string> &args); uint64_t get_daemon_blockchain_height(std::string& err); diff --git a/src/version.cmake b/src/version.cmake index 14ce8843d..ef4eb1fc9 100644 --- a/src/version.cmake +++ b/src/version.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -80,4 +80,4 @@ else() endif() configure_file("src/version.h.in" "${TO}") -endif()
\ No newline at end of file +endif() diff --git a/src/version.h.in b/src/version.h.in index a09131cdb..6dd62faf9 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -1,3 +1,3 @@ #define MONERO_VERSION_TAG "@VERSIONTAG@" -#define MONERO_VERSION "0.8.8.6" +#define MONERO_VERSION "0.8.8.7" #define MONERO_VERSION_FULL MONERO_VERSION "-" MONERO_VERSION_TAG diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index af3ec0fb8..6cf496f0d 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 163e19df4..6698e7296 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -227,24 +227,25 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ } tx_extra_nonce extra_nonce; + crypto::hash payment_id = null_hash; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash payment_id; if(get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { - uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0; - if (0 < received && null_hash != payment_id) - { - payment_details payment; - payment.m_tx_hash = cryptonote::get_transaction_hash(tx); - payment.m_amount = received; - payment.m_block_height = height; - payment.m_unlock_time = tx.unlock_time; - m_payments.emplace(payment_id, payment); - LOG_PRINT_L2("Payment found: " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount); - } + // We got a payment ID to go with this tx } } + uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0; + if (0 < received) + { + payment_details payment; + payment.m_tx_hash = cryptonote::get_transaction_hash(tx); + payment.m_amount = received; + payment.m_block_height = height; + payment.m_unlock_time = tx.unlock_time; + m_payments.emplace(payment_id, payment); + LOG_PRINT_L2("Payment found: " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount); + } } //---------------------------------------------------------------------------------------------------- void wallet2::process_unconfirmed(const cryptonote::transaction& tx) @@ -571,6 +572,54 @@ void wallet2::load_keys(const std::string& keys_file_name, const std::string& pa } /*! + * \brief verify password for default wallet keys file. + * \param password Password to verify + * + * for verification only + * should not mutate state, unlike load_keys() + * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password + * + */ +bool wallet2::verify_password(const std::string& password) +{ + const std::string keys_file_name = m_keys_file; + wallet2::keys_file_data keys_file_data; + std::string buf; + bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); + THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); + + // Decrypt the contents + r = ::serialization::parse_binary(buf, keys_file_data); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); + crypto::chacha8_key key; + crypto::generate_chacha8_key(password, key); + std::string account_data; + account_data.resize(keys_file_data.account_data.size()); + crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + + // The contents should be JSON if the wallet follows the new format. + rapidjson::Document json; + if (json.Parse(account_data.c_str(), keys_file_data.account_data.size()).HasParseError()) + { + // old format before JSON wallet key file format + } + else + { + account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() + + json["key_data"].GetStringLength()); + } + + cryptonote::account_base account_data_check; + + r = epee::serialization::load_t_from_binary(account_data_check, account_data); + const cryptonote::account_keys& keys = account_data_check.get_keys(); + + r = r && verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); + r = r && verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); + return r; +} + +/*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file * \param password Password of wallet file @@ -616,11 +665,9 @@ void wallet2::rewrite(const std::string& wallet_name, const std::string& passwor { prepare_file_names(wallet_name); boost::system::error_code ignored_ec; - THROW_WALLET_EXCEPTION_IF(!boost::filesystem::exists(m_wallet_file, ignored_ec), error::file_not_found, m_wallet_file); THROW_WALLET_EXCEPTION_IF(!boost::filesystem::exists(m_keys_file, ignored_ec), error::file_not_found, m_keys_file); bool r = store_keys(m_keys_file, password); THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - store(); } //---------------------------------------------------------------------------------------------------- void wallet2::wallet_exists(const std::string& file_path, bool& keys_file_exists, bool& wallet_file_exists) @@ -633,6 +680,11 @@ void wallet2::wallet_exists(const std::string& file_path, bool& keys_file_exists wallet_file_exists = boost::filesystem::exists(wallet_file, ignore); } //---------------------------------------------------------------------------------------------------- +bool wallet2::wallet_valid_path_format(const std::string& file_path) +{ + return !file_path.empty(); +} +//---------------------------------------------------------------------------------------------------- bool wallet2::parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id) { cryptonote::blobdata payment_id_data; @@ -686,14 +738,16 @@ void wallet2::load(const std::string& wallet_, const std::string& password) { LOG_PRINT_L0("file not found: " << m_wallet_file << ", starting with empty blockchain"); m_account_public_address = m_account.get_keys().m_account_address; - return; } - bool r = tools::unserialize_obj_from_file(*this, m_wallet_file); - THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); - THROW_WALLET_EXCEPTION_IF( - m_account_public_address.m_spend_public_key != m_account.get_keys().m_account_address.m_spend_public_key || - m_account_public_address.m_view_public_key != m_account.get_keys().m_account_address.m_view_public_key, - error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); + else + { + bool r = tools::unserialize_obj_from_file(*this, m_wallet_file); + THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); + THROW_WALLET_EXCEPTION_IF( + m_account_public_address.m_spend_public_key != m_account.get_keys().m_account_address.m_spend_public_key || + m_account_public_address.m_view_public_key != m_account.get_keys().m_account_address.m_view_public_key, + error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); + } cryptonote::block genesis; generate_genesis(genesis); @@ -763,6 +817,17 @@ void wallet2::get_payments(const crypto::hash& payment_id, std::list<wallet2::pa }); } //---------------------------------------------------------------------------------------------------- +void wallet2::get_payments(std::list<std::pair<crypto::hash,wallet2::payment_details>>& payments, uint64_t min_height) const +{ + auto range = std::make_pair(m_payments.begin(), m_payments.end()); + std::for_each(range.first, range.second, [&payments, &min_height](const payment_container::value_type& x) { + if (min_height < x.second.m_block_height) + { + payments.push_back(x); + } + }); +} +//---------------------------------------------------------------------------------------------------- bool wallet2::is_transfer_unlocked(const transfer_details& td) const { if(!is_tx_spendtime_unlocked(td.m_tx.unlock_time)) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index f22c5d79d..1febfba39 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -82,7 +82,7 @@ namespace tools { wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false) {}; public: - wallet2(bool testnet = false) : m_run(true), m_callback(0), m_testnet(testnet), is_old_file_format(false) {}; + wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_restricted(restricted), is_old_file_format(false) {}; struct transfer_details { uint64_t m_block_height; @@ -153,6 +153,11 @@ namespace tools void rewrite(const std::string& wallet_name, const std::string& password); void load(const std::string& wallet, const std::string& password); void store(); + + /*! + * \brief verifies given password is correct for default wallet keys file + */ + bool verify_password(const std::string& password); cryptonote::account_base& get_account(){return m_account;} // upper_transaction_size_limit as defined below is set to @@ -191,6 +196,7 @@ namespace tools bool refresh(size_t & blocks_fetched, bool& received_money, bool& ok); bool testnet() { return m_testnet; } + bool restricted() const { return m_restricted; } uint64_t balance(); uint64_t unlocked_balance(); @@ -206,6 +212,7 @@ namespace tools bool check_connection(); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0) const; + void get_payments(std::list<std::pair<crypto::hash,wallet2::payment_details>>& payments, uint64_t min_height) const; uint64_t get_blockchain_current_height() const { return m_local_bc_height; } template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) @@ -224,7 +231,19 @@ namespace tools a & m_payments; } + /*! + * \brief Check if wallet keys and bin files exist + * \param file_path Wallet file path + * \param keys_file_exists Whether keys file exists + * \param wallet_file_exists Whether bin file exists + */ static void wallet_exists(const std::string& file_path, bool& keys_file_exists, bool& wallet_file_exists); + /*! + * \brief Check if wallet file path is valid format + * \param file_path Wallet file path + * \return Whether path is valid format + */ + static bool wallet_valid_path_format(const std::string& file_path); static bool parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id); @@ -279,6 +298,7 @@ namespace tools i_wallet2_callback* m_callback; bool m_testnet; + bool m_restricted; std::string seed_language; /*!< Language of the mnemonics (seed). */ bool is_old_file_format; /*!< Whether the wallet file is of an old file format */ }; diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index ebec931e4..bf520c36c 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 66cf64166..e39d9ba7b 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -85,7 +85,7 @@ namespace tools return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init(m_port, m_bind_ip); } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er) { try { @@ -101,7 +101,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er) { try { @@ -161,12 +161,19 @@ namespace tools } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er) { std::vector<cryptonote::tx_destination_entry> dsts; std::vector<uint8_t> extra; + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + // validate the transfer requested and populate dsts & extra if (!validate_transfer(req.destinations, req.payment_id, dsts, extra, er)) { @@ -212,12 +219,19 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er) { std::vector<cryptonote::tx_destination_entry> dsts; std::vector<uint8_t> extra; + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + // validate the transfer requested and populate dsts & extra; RPC_TRANSFER::request and RPC_TRANSFER_SPLIT::request are identical types. if (!validate_transfer(req.destinations, req.payment_id, dsts, extra, er)) { @@ -259,8 +273,15 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er) { + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + try { m_wallet.store(); @@ -274,7 +295,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er) { crypto::hash payment_id; cryptonote::blobdata payment_id_blob; @@ -311,10 +332,30 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er) { res.payments.clear(); + /* If the payment ID list is empty, we get payments to any payment ID (or lack thereof) */ + if (req.payment_ids.empty()) + { + std::list<std::pair<crypto::hash,wallet2::payment_details>> payment_list; + m_wallet.get_payments(payment_list, req.min_block_height); + + for (auto & payment : payment_list) + { + wallet_rpc::payment_details rpc_payment; + rpc_payment.payment_id = epee::string_tools::pod_to_hex(payment.first); + rpc_payment.tx_hash = epee::string_tools::pod_to_hex(payment.second.m_tx_hash); + rpc_payment.amount = payment.second.m_amount; + rpc_payment.block_height = payment.second.m_block_height; + rpc_payment.unlock_time = payment.second.m_unlock_time; + res.payments.push_back(std::move(rpc_payment)); + } + + return true; + } + for (auto & payment_id_str : req.payment_ids) { crypto::hash payment_id; @@ -356,7 +397,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er) { if(req.transfer_type.compare("all") != 0 && req.transfer_type.compare("available") != 0 && req.transfer_type.compare("unavailable") != 0) { @@ -390,11 +431,13 @@ namespace tools { transfers_found = true; } + auto txBlob = t_serializable_object_to_blob(td.m_tx); wallet_rpc::transfer_details rpc_transfers; rpc_transfers.amount = td.amount(); rpc_transfers.spent = td.m_spent; rpc_transfers.global_index = td.m_global_output_index; rpc_transfers.tx_hash = boost::lexical_cast<std::string>(cryptonote::get_transaction_hash(td.m_tx)); + rpc_transfers.tx_size = txBlob.size(); res.transfers.push_back(rpc_transfers); } } @@ -407,8 +450,15 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er) { + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (req.key_type.compare("mnemonic") == 0) { if (!m_wallet.get_seed(res.key)) diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 6f52e87ba..cfdc1c147 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -74,20 +74,20 @@ namespace tools END_URI_MAP2() //json_rpc - bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er, connection_context& cntx); + bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er); + bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er); bool validate_transfer(const std::list<wallet_rpc::transfer_destination> destinations, const std::string payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, epee::json_rpc::error& er); - bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er, connection_context& cntx); + bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er); + bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er); + bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er); + bool on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er); + bool on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er); + bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er); bool handle_command_line(const boost::program_options::variables_map& vm); //json rpc v2 - bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, connection_context& cntx); + bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er); wallet2& m_wallet; std::string m_port; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 48f8ec2a5..35783a189 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -228,12 +228,14 @@ namespace wallet_rpc bool spent; uint64_t global_index; std::string tx_hash; + uint64_t tx_size; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) KV_SERIALIZE(spent) KV_SERIALIZE(global_index) KV_SERIALIZE(tx_hash) + KV_SERIALIZE(tx_size) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index 75f5e2b9c..28642c19d 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -37,3 +37,4 @@ #define WALLET_RPC_ERROR_CODE_GENERIC_TRANSFER_ERROR -4 #define WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID -5 #define WALLET_RPC_ERROR_CODE_TRANSFER_TYPE -6 +#define WALLET_RPC_ERROR_CODE_DENIED -7 |