diff options
Diffstat (limited to 'src')
361 files changed, 17105 insertions, 3427 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a0b62da77..da6d76d97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -109,7 +109,9 @@ add_subdirectory(ringct) add_subdirectory(checkpoints) add_subdirectory(cryptonote_basic) add_subdirectory(cryptonote_core) +add_subdirectory(lmdb) add_subdirectory(multisig) +add_subdirectory(net) if(NOT IOS) add_subdirectory(blockchain_db) endif() diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 277f4458e..db161fa5e 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 60a7326f8..3eb24494f 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index e80adae9e..04a33d7c6 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index c25798c1e..d772bf4bb 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -121,8 +121,10 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) { + const transaction &tx = txp.first; + bool miner_tx = false; crypto::hash tx_hash, tx_prunable_hash; if (!tx_hash_ptr) @@ -138,7 +140,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti if (tx.version >= 2) { if (!tx_prunable_hash_ptr) - tx_prunable_hash = get_transaction_prunable_hash(tx); + tx_prunable_hash = get_transaction_prunable_hash(tx, &txp.second); else tx_prunable_hash = *tx_prunable_hash_ptr; } @@ -168,7 +170,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash, tx_prunable_hash); + uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash); std::vector<uint64_t> amount_output_indices(tx.vout.size()); @@ -195,13 +197,16 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti add_tx_amount_output_indices(tx_id, amount_output_indices); } -uint64_t BlockchainDB::add_block( const block& blk +uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector<transaction>& txs + , const std::vector<std::pair<transaction, blobdata>>& txs ) { + const block &blk = blck.first; + // sanity if (blk.tx_hashes.size() != txs.size()) throw std::runtime_error("Inconsistent tx/hashes sizes"); @@ -220,16 +225,16 @@ uint64_t BlockchainDB::add_block( const block& blk time1 = epee::misc_utils::get_tick_count(); uint64_t num_rct_outs = 0; - add_transaction(blk_hash, blk.miner_tx); + add_transaction(blk_hash, std::make_pair(blk.miner_tx, tx_to_blob(blk.miner_tx))); if (blk.miner_tx.version == 2) num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; - for (const transaction& tx : txs) + for (const std::pair<transaction, blobdata>& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); - for (const auto &vout: tx.vout) + for (const auto &vout: tx.first.vout) { if (vout.amount == 0) ++num_rct_outs; @@ -241,7 +246,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); + add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; @@ -267,7 +272,10 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs) for (const auto& h : boost::adaptors::reverse(blk.tx_hashes)) { - txs.push_back(get_tx(h)); + cryptonote::transaction tx; + if (!get_tx(h, tx) && !get_pruned_tx(h, tx)) + throw DB_ERROR("Failed to get pruned or unpruned transaction from the db"); + txs.push_back(std::move(tx)); remove_transaction(h); } remove_transaction(get_transaction_hash(blk.miner_tx)); @@ -280,7 +288,7 @@ bool BlockchainDB::is_open() const void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { - transaction tx = get_tx(tx_hash); + transaction tx = get_pruned_tx(tx_hash); for (const txin_v& tx_input : tx.vin) { @@ -325,6 +333,17 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co return true; } +bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const +{ + blobdata bd; + if (!get_pruned_tx_blob(h, bd)) + return false; + if (!parse_and_validate_tx_base_from_blob(bd, tx)) + throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db"); + + return true; +} + transaction BlockchainDB::get_tx(const crypto::hash& h) const { transaction tx; @@ -333,6 +352,14 @@ transaction BlockchainDB::get_tx(const crypto::hash& h) const return tx; } +transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const +{ + transaction tx; + if (!get_pruned_tx(h, tx)) + throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()); + return tx; +} + void BlockchainDB::reset_stats() { num_calls = 0; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 9d4f59b3c..313e716e0 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -358,12 +358,14 @@ private: * * @param blk the block to be added * @param block_weight the weight of the block (transactions and all) + * @param long_term_block_weight the long term block weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -375,7 +377,7 @@ private: * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -402,7 +404,7 @@ private: * @param tx_prunable_hash the hash of the prunable part of the transaction * @return the transaction ID */ - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; /** * @brief remove data about a transaction @@ -530,7 +532,7 @@ protected: * @param tx_hash_ptr the hash of the transaction, if already calculated * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated */ - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); + void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric @@ -789,17 +791,19 @@ public: * * @param blk the block to be added * @param block_weight the size of the block (transactions and all) + * @param long_term_block_weight the long term weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block * * @return the height of the chain post-addition */ - virtual uint64_t add_block( const block& blk + virtual uint64_t add_block( const std::pair<block, blobdata>& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector<transaction>& txs + , const std::vector<std::pair<transaction, blobdata>>& txs ); /** @@ -985,6 +989,17 @@ public: virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; /** + * @brief fetch a block's long term weight + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param height the height requested + * + * @return the long term weight + */ + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0; + + /** * @brief fetch a block's hash * * The subclass should return hash of the block with the @@ -1037,9 +1052,11 @@ public: * * The subclass should return the hash of the most recent block * + * @param block_height if non NULL, returns the height of that block (ie, the blockchain height minus 1) + * * @return the top block's hash */ - virtual crypto::hash top_block_hash() const = 0; + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const = 0; /** * @brief fetch the top block @@ -1126,6 +1143,17 @@ public: virtual transaction get_tx(const crypto::hash& h) const; /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should throw TX_DNE. + * + * @param h the hash to look for + * + * @return the transaction with the given hash + */ + virtual transaction get_pruned_tx(const crypto::hash& h) const; + + /** * @brief fetches the transaction with the given hash * * If the transaction does not exist, the subclass should return false. @@ -1137,6 +1165,17 @@ public: virtual bool get_tx(const crypto::hash& h, transaction &tx) const; /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found + */ + virtual bool get_pruned_tx(const crypto::hash& h, transaction &tx) const; + + /** * @brief fetches the transaction blob with the given hash * * The subclass should return the transaction stored which has the given @@ -1165,6 +1204,21 @@ public: virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; /** + * @brief fetches the prunable transaction blob with the given hash + * + * The subclass should return the prunable transaction stored which has the given + * hash. + * + * If the transaction does not exist, or if we do not have that prunable data, + * the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found and we have its prunable data + */ + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; + + /** * @brief fetches the prunable transaction hash * * The subclass should return the hash of the prunable transaction data. @@ -1258,7 +1312,7 @@ public: * * @return the requested output data */ - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt = true) const = 0; /** * @brief gets an output's tx hash and index @@ -1329,10 +1383,11 @@ public: * If an output cannot be found, the subclass should throw OUTPUT_DNE. * * @param tx_id a transaction ID + * @param n_txes how many txes to get data for, starting with tx_id * * @return a list of amount-specific output indices */ - virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0; + virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes = 1) const = 0; /** * @brief check if a key image is stored as spent @@ -1412,6 +1467,31 @@ public: virtual void prune_outputs(uint64_t amount) = 0; /** + * @brief get the blockchain pruning seed + * @return the blockchain pruning seed + */ + virtual uint32_t get_blockchain_pruning_seed() const = 0; + + /** + * @brief prunes the blockchain + * @param pruning_seed the seed to use, 0 for default (highly recommended) + * @return success iff true + */ + virtual bool prune_blockchain(uint32_t pruning_seed = 0) = 0; + + /** + * @brief prunes recent blockchain changes as needed, iff pruning is enabled + * @return success iff true + */ + virtual bool update_pruning() = 0; + + /** + * @brief checks pruning was done correctly, iff enabled + * @return success iff true + */ + virtual bool check_pruning() = 0; + + /** * @brief runs a function over all txpool transactions * * The subclass should run the passed function for each txpool tx it has diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h index b8c7fa3e3..04cadbb10 100644 --- a/src/blockchain_db/db_types.h +++ b/src/blockchain_db/db_types.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 77f8ade7f..becffed16 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -29,12 +29,14 @@ #include <boost/filesystem.hpp> #include <boost/format.hpp> +#include <boost/circular_buffer.hpp> #include <memory> // std::unique_ptr #include <cstring> // memcpy #include "string_tools.h" #include "file_io_utils.h" #include "common/util.h" +#include "common/pruning.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "crypto/crypto.h" #include "profile_tools.h" @@ -52,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 3 +#define VERSION 4 namespace { @@ -130,14 +132,20 @@ private: std::unique_ptr<char[]> data; }; -int compare_uint64(const MDB_val *a, const MDB_val *b) +} + +namespace cryptonote { - const uint64_t va = *(const uint64_t *)a->mv_data; - const uint64_t vb = *(const uint64_t *)b->mv_data; + +int BlockchainLMDB::compare_uint64(const MDB_val *a, const MDB_val *b) +{ + uint64_t va, vb; + memcpy(&va, a->mv_data, sizeof(va)); + memcpy(&vb, b->mv_data, sizeof(vb)); return (va < vb) ? -1 : va > vb; } -int compare_hash32(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_hash32(const MDB_val *a, const MDB_val *b) { uint32_t *va = (uint32_t*) a->mv_data; uint32_t *vb = (uint32_t*) b->mv_data; @@ -151,13 +159,18 @@ int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } -int compare_string(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b) { const char *va = (const char*) a->mv_data; const char *vb = (const char*) b->mv_data; return strcmp(va, vb); } +} + +namespace +{ + /* DB schema: * * Table Key Data @@ -169,6 +182,7 @@ int compare_string(const MDB_val *a, const MDB_val *b) * txs_pruned txn ID pruned txn blob * txs_prunable txn ID prunable txn blob * txs_prunable_hash txn ID prunable txn hash + * txs_prunable_tip txn ID height * tx_indices txn hash {txn ID, metadata} * tx_outputs txn ID [txn amount output indices] * @@ -196,6 +210,7 @@ const char* const LMDB_TXS = "txs"; const char* const LMDB_TXS_PRUNED = "txs_pruned"; const char* const LMDB_TXS_PRUNABLE = "txs_prunable"; const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash"; +const char* const LMDB_TXS_PRUNABLE_TIP = "txs_prunable_tip"; const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -253,7 +268,17 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { -typedef struct mdb_block_info_old +typedef struct mdb_block_info_1 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; +} mdb_block_info_1; + +typedef struct mdb_block_info_2 { uint64_t bi_height; uint64_t bi_timestamp; @@ -261,9 +286,10 @@ typedef struct mdb_block_info_old uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; -} mdb_block_info_old; + uint64_t bi_cum_rct; +} mdb_block_info_2; -typedef struct mdb_block_info +typedef struct mdb_block_info_3 { uint64_t bi_height; uint64_t bi_timestamp; @@ -272,18 +298,16 @@ typedef struct mdb_block_info difficulty_type bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; -} mdb_block_info; + uint64_t bi_long_term_block_weight; +} mdb_block_info_3; + +typedef mdb_block_info_3 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; uint64_t bh_height; } blk_height; -typedef struct txindex { - crypto::hash key; - tx_data_t data; -} txindex; - typedef struct pre_rct_outkey { uint64_t amount_index; uint64_t output_id; @@ -495,7 +519,7 @@ void BlockchainLMDB::do_resize(uint64_t increase_size) mdb_env_stat(m_env, &mst); // add 1Gb per resize, instead of doing a percentage increase - uint64_t new_mapsize = (double) mei.me_mapsize + add_size; + uint64_t new_mapsize = (uint64_t) mei.me_mapsize + add_size; // If given, use increase_size instead of above way of resizing. // This is currently used for increasing by an estimated size at start of new @@ -549,18 +573,18 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const // additional size needed. uint64_t size_used = mst.ms_psize * mei.me_last_pgno; - LOG_PRINT_L1("DB map size: " << mei.me_mapsize); - LOG_PRINT_L1("Space used: " << size_used); - LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); - LOG_PRINT_L1("Size threshold: " << threshold_size); + MDEBUG("DB map size: " << mei.me_mapsize); + MDEBUG("Space used: " << size_used); + MDEBUG("Space remaining: " << mei.me_mapsize - size_used); + MDEBUG("Size threshold: " << threshold_size); float resize_percent = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); + MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { if (mei.me_mapsize - size_used < threshold_size) { - LOG_PRINT_L1("Threshold met (size-based)"); + MINFO("Threshold met (size-based)"); return true; } else @@ -569,7 +593,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const if ((double)size_used / mei.me_mapsize > resize_percent) { - LOG_PRINT_L1("Threshold met (percent-based)"); + MINFO("Threshold met (percent-based)"); return true; } return false; @@ -581,7 +605,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size"); + MTRACE("[" << __func__ << "] " << "checking DB size"); const uint64_t min_increase_size = 512 * (1 << 20); uint64_t threshold_size = 0; uint64_t increase_size = 0; @@ -685,7 +709,7 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -745,6 +769,7 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const diff const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; bi.bi_cum_rct += bi_prev->bi_cum_rct; } + bi.bi_long_term_block_weight = long_term_block_weight; MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); @@ -798,7 +823,7 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } -uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -811,6 +836,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_indices) MDB_val_set(val_tx_id, tx_id); @@ -823,6 +849,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); } + const cryptonote::transaction &tx = txp.first; txindex ti; ti.key = tx_hash; ti.data.tx_id = tx_id; @@ -836,28 +863,41 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); - cryptonote::blobdata blob = tx_to_blob(tx); + const cryptonote::blobdata &blob = txp.second; MDB_val_sized(blobval, blob); - std::stringstream ss; - binary_archive<true> ba(ss); - bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); - if (!r) - throw0(DB_ERROR("Failed to serialize pruned tx")); - std::string pruned = ss.str(); - MDB_val_sized(pruned_blob, pruned); + unsigned int unprunable_size = tx.unprunable_size; + if (unprunable_size == 0) + { + std::stringstream ss; + binary_archive<true> ba(ss); + bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); + if (!r) + throw0(DB_ERROR("Failed to serialize pruned tx")); + unprunable_size = ss.str().size(); + } + + if (unprunable_size > blob.size()) + throw0(DB_ERROR("pruned tx size is larger than tx size")); + + MDB_val pruned_blob = {unprunable_size, (void*)blob.data()}; result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); - if (pruned.size() > blob.size()) - throw0(DB_ERROR("pruned tx size is larger than tx size")); - cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size()); - MDB_val_sized(prunable_blob, prunable); + MDB_val prunable_blob = {blob.size() - unprunable_size, (void*)(blob.data() + unprunable_size)}; result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); + if (get_blockchain_pruning_seed()) + { + MDB_val_set(val_height, m_height); + result = mdb_cursor_put(m_cur_txs_prunable_tip, &val_tx_id, &val_height, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str())); + } + if (tx.version > 1) { MDB_val_set(val_prunable_hash, tx_prunable_hash); @@ -883,6 +923,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_outputs) MDB_val_set(val_h, tx_hash); @@ -898,11 +939,25 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET))) + result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + } + else if (result != MDB_NOTFOUND) throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str())); - result = mdb_cursor_del(m_cur_txs_prunable, 0); - if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + + result = mdb_cursor_get(m_cur_txs_prunable_tip, &val_tx_id, NULL, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Failed to locate tx id for removal: ", result).c_str())); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable_tip, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str())); + } if (tx.version > 1) { @@ -1025,7 +1080,8 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector<uint64_t> amount_output_indices = get_tx_amount_output_indices(tx_id); + std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1); + const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front(); if (amount_output_indices.empty()) { @@ -1279,14 +1335,14 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) MDB_envinfo mei; mdb_env_info(m_env, &mei); - uint64_t cur_mapsize = (double)mei.me_mapsize; + uint64_t cur_mapsize = (uint64_t)mei.me_mapsize; if (cur_mapsize < mapsize) { if (auto result = mdb_env_set_mapsize(m_env, mapsize)) throw0(DB_ERROR(lmdb_error("Failed to set max memory map size: ", result).c_str())); mdb_env_info(m_env, &mei); - cur_mapsize = (double)mei.me_mapsize; + cur_mapsize = (uint64_t)mei.me_mapsize; LOG_PRINT_L1("LMDB memory map size: " << cur_mapsize); } @@ -1307,6 +1363,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // open necessary databases, and set properties as needed // uses macros to avoid having to change things too many places + // also change blockchain_prune.cpp to match lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); @@ -1315,7 +1372,9 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned"); lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable"); - lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + if (!(mdb_flags & MDB_RDONLY)) + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_TIP, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_tip, "Failed to open db handle for m_txs_prunable_tip"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); @@ -1343,6 +1402,10 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_output_txs, compare_uint64); mdb_set_dupsort(txn, m_block_info, compare_uint64); + if (!(mdb_flags & MDB_RDONLY)) + mdb_set_dupsort(txn, m_txs_prunable_tip, compare_uint64); + mdb_set_compare(txn, m_txs_prunable, compare_uint64); + mdb_set_dupsort(txn, m_txs_prunable_hash, compare_uint64); mdb_set_compare(txn, m_txpool_meta, compare_hash32); mdb_set_compare(txn, m_txpool_blob, compare_hash32); @@ -1501,6 +1564,8 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str())); if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_prunable_tip, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_tip: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_indices, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_outputs, 0)) @@ -1826,6 +1891,290 @@ cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid return bd; } +uint32_t BlockchainLMDB::get_blockchain_pruning_seed() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(properties) + MDB_val_str(k, "pruning_seed"); + MDB_val v; + int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return 0; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning seed: ", result).c_str())); + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + uint32_t pruning_seed; + memcpy(&pruning_seed, v.mv_data, sizeof(pruning_seed)); + TXN_POSTFIX_RDONLY(); + return pruning_seed; +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to find transaction pruned data: ", ret).c_str())); + if (v.mv_size == 0) + throw0(DB_ERROR("Invalid transaction pruned data")); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +enum { prune_mode_prune, prune_mode_update, prune_mode_check }; + +bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + const uint32_t log_stripes = tools::get_pruning_log_stripes(pruning_seed); + if (log_stripes && log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Pruning seed not in range")); + pruning_seed = tools::get_pruning_stripe(pruning_seed);; + if (pruning_seed > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + throw0(DB_ERROR("Pruning seed not in range")); + check_open(); + + TIME_MEASURE_START(t); + + size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0; + uint64_t n_bytes = 0; + + mdb_txn_safe txn; + auto result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages0 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + + MDB_val_str(k, "pruning_seed"); + MDB_val v; + result = mdb_get(txn, m_properties, &k, &v); + bool prune_tip_table = false; + if (result == MDB_NOTFOUND) + { + // not pruned yet + if (mode != prune_mode_prune) + { + txn.abort(); + TIME_MEASURE_FINISH(t); + MDEBUG("Pruning not enabled, nothing to do"); + return true; + } + if (pruning_seed == 0) + pruning_seed = tools::get_random_stripe(); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + v.mv_data = &pruning_seed; + v.mv_size = sizeof(pruning_seed); + result = mdb_put(txn, m_properties, &k, &v, 0); + if (result) + throw0(DB_ERROR("Failed to save pruning seed")); + prune_tip_table = false; + } + else if (result == 0) + { + // pruned already + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + const uint32_t data = *(const uint32_t*)v.mv_data; + if (pruning_seed == 0) + pruning_seed = tools::get_pruning_stripe(data); + if (tools::get_pruning_stripe(data) != pruning_seed) + throw0(DB_ERROR("Blockchain already pruned with different seed")); + if (tools::get_pruning_log_stripes(data) != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Blockchain already pruned with different base")); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + prune_tip_table = (mode == prune_mode_update); + } + else + { + throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str())); + } + + if (mode == prune_mode_check) + MINFO("Checking blockchain pruning..."); + else + MINFO("Pruning blockchain..."); + + MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip; + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + const uint64_t blockchain_height = height(); + + if (prune_tip_table) + { + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_txs_prunable_tip, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + uint64_t block_height; + memcpy(&block_height, v.mv_data, sizeof(block_height)); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS < blockchain_height) + { + ++n_total_records; + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &k)) + { + ++n_prunable_records; + result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str())); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += k.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + result = mdb_cursor_del(c_txs_prunable_tip, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str())); + } + } + } + else + { + MDB_cursor *c_tx_indices; + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + ++n_total_records; + //const txindex *ti = (const txindex *)v.mv_data; + txindex ti; + memcpy(&ti, v.mv_data, sizeof(ti)); + const uint64_t block_height = ti.data.block_id; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDB_val_set(kp, ti.data.tx_id); + MDB_val_set(vp, block_height); + if (mode == prune_mode_check) + { + result = mdb_cursor_get(c_txs_prunable_tip, &kp, &vp, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Transaction not found in prunable tip table for height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + result = mdb_cursor_put(c_txs_prunable_tip, &kp, &vp, 0); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + } + } + MDB_val_set(kp, ti.data.tx_id); + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &kp)) + { + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (mode == prune_mode_check) + { + if (result != MDB_NOTFOUND) + MERROR("Prunable data found for pruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + ++n_prunable_records; + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += kp.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + } + else + { + if (mode == prune_mode_check) + { + MDB_val_set(kp, ti.data.tx_id); + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Prunable data not found for unpruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + } + } + mdb_cursor_close(c_tx_indices); + } + + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages1 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + const size_t db_bytes = (pages0 - pages1) * db_stats.ms_psize; + + mdb_cursor_close(c_txs_prunable_tip); + mdb_cursor_close(c_txs_prunable); + mdb_cursor_close(c_txs_pruned); + + txn.commit(); + + TIME_MEASURE_FINISH(t); + + MINFO((mode == prune_mode_check ? "Checked" : "Pruned") << " blockchain in " << + t << " ms: " << (n_bytes/1024.0f/1024.0f) << " MB (" << db_bytes/1024.0f/1024.0f << " MB) pruned in " << + n_pruned_records << " records (" << pages0 - pages1 << "/" << pages0 << " " << db_stats.ms_psize << " byte pages), " << + n_prunable_records << "/" << n_total_records << " pruned records"); + return true; +} + +bool BlockchainLMDB::prune_blockchain(uint32_t pruning_seed) +{ + return prune_worker(prune_mode_prune, pruning_seed); +} + +bool BlockchainLMDB::update_pruning() +{ + return prune_worker(prune_mode_update, 0); +} + +bool BlockchainLMDB::check_pruning() +{ + return prune_worker(prune_mode_check, 0); +} + bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2159,6 +2508,29 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh return ret; } +uint64_t BlockchainLMDB::get_block_long_term_weight(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get block long term weight from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block info not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a long term block weight from the db")); + + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_long_term_block_weight; + TXN_POSTFIX_RDONLY(); + return ret; +} + crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2210,11 +2582,13 @@ std::vector<crypto::hash> BlockchainLMDB::get_hashes_range(const uint64_t& h1, c return v; } -crypto::hash BlockchainLMDB::top_block_hash() const +crypto::hash BlockchainLMDB::top_block_hash(uint64_t *block_height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); uint64_t m_height = height(); + if (block_height) + *block_height = m_height - 1; if (m_height != 0) { return get_block_hash_from_height(m_height - 1); @@ -2427,6 +2801,36 @@ bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobd return true; } +bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_prunable); + + MDB_val_set(v, h); + MDB_val result; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + const txindex *tip = (const txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result, MDB_SET); + } + if (get_result == MDB_NOTFOUND) + return false; + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + TXN_POSTFIX_RDONLY(); + + return true; +} + bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2535,7 +2939,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2562,7 +2966,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; memcpy(&ret, &okp->data, sizeof(pre_rct_output_data_t));; - ret.commitment = rct::zeroCommit(amount); + if (include_commitmemt) + ret.commitment = rct::zeroCommit(amount); } TXN_POSTFIX_RDONLY(); return ret; @@ -2604,7 +3009,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const +std::vector<std::vector<uint64_t>> BlockchainLMDB::get_tx_amount_output_indices(uint64_t tx_id, size_t n_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2613,35 +3018,40 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_ TXN_PREFIX_RDONLY(); RCURSOR(tx_outputs); - int result = 0; MDB_val_set(k_tx_id, tx_id); MDB_val v; - std::vector<uint64_t> amount_output_indices; + std::vector<std::vector<uint64_t>> amount_output_indices_set; + amount_output_indices_set.reserve(n_txes); - result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); - if (result == MDB_NOTFOUND) - LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " - "tx_outputs, but it should have an empty entry even if it's a tx without " - "outputs"); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); + MDB_cursor_op op = MDB_SET; + while (n_txes-- > 0) + { + int result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, op); + if (result == MDB_NOTFOUND) + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " + "tx_outputs, but it should have an empty entry even if it's a tx without " + "outputs"); + else if (result) + throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); - const uint64_t* indices = (const uint64_t*)v.mv_data; - int num_outputs = v.mv_size / sizeof(uint64_t); + op = MDB_NEXT; - amount_output_indices.reserve(num_outputs); - for (int i = 0; i < num_outputs; ++i) - { - // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); - amount_output_indices.push_back(indices[i]); + const uint64_t* indices = (const uint64_t*)v.mv_data; + size_t num_outputs = v.mv_size / sizeof(uint64_t); + + amount_output_indices_set.resize(amount_output_indices_set.size() + 1); + std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.back(); + amount_output_indices.reserve(num_outputs); + for (size_t i = 0; i < num_outputs; ++i) + { + amount_output_indices.push_back(indices[i]); + } } - indices = nullptr; TXN_POSTFIX_RDONLY(); - return amount_output_indices; + return amount_output_indices_set; } - bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3161,8 +3571,8 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - const std::vector<transaction>& txs) +uint64_t BlockchainLMDB::add_block(const std::pair<block, blobdata>& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, + const std::vector<std::pair<transaction, blobdata>>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -3180,7 +3590,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const try { - BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -3724,7 +4134,7 @@ void BlockchainLMDB::migrate_0_1() break; } MDB_dbi diffs, hashes, sizes, timestamps; - mdb_block_info_old bi; + mdb_block_info_1 bi; MDB_val_set(nv, bi); lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); @@ -4081,7 +4491,7 @@ void BlockchainLMDB::migrate_0_1() if (!parse_and_validate_block_from_blob(bd, b)) throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - add_transaction(null_hash, b.miner_tx); + add_transaction(null_hash, std::make_pair(b.miner_tx, tx_to_blob(b.miner_tx))); for (unsigned int j = 0; j<b.tx_hashes.size(); j++) { transaction tx; hk.mv_data = &b.tx_hashes[j]; @@ -4091,7 +4501,7 @@ void BlockchainLMDB::migrate_0_1() bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - add_transaction(null_hash, tx, &b.tx_hashes[j]); + add_transaction(null_hash, std::make_pair(std::move(tx), bd), &b.tx_hashes[j]); result = mdb_cursor_del(c_txs, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str())); @@ -4351,8 +4761,8 @@ void BlockchainLMDB::migrate_2_3() } else if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); - const mdb_block_info_old *bi_old = (const mdb_block_info_old*)v.mv_data; - mdb_block_info bi; + const mdb_block_info_1 *bi_old = (const mdb_block_info_1*)v.mv_data; + mdb_block_info_2 bi; bi.bi_height = bi_old->bi_height; bi.bi_timestamp = bi_old->bi_timestamp; bi.bi_coins = bi_old->bi_coins; @@ -4385,6 +4795,7 @@ void BlockchainLMDB::migrate_2_3() throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); mdb_set_dupsort(txn, m_block_info, compare_uint64); @@ -4405,6 +4816,166 @@ void BlockchainLMDB::migrate_2_3() txn.commit(); } +void BlockchainLMDB::migrate_3_4() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + bool past_long_term_weight = false; + + MGINFO_YELLOW("Migrating blockchain from DB version 3 to 4 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + boost::circular_buffer<uint64_t> long_term_block_weights(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE); + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_2 *bi_old = (const mdb_block_info_2*)v.mv_data; + mdb_block_info_3 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + + // get block major version to determine which rule is in place + if (!past_long_term_weight) + { + MDB_val_copy<uint64_t> kb(bi.bi_height); + MDB_val vb; + result = mdb_cursor_get(c_blocks, &kb, &vb, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + if (vb.mv_size == 0) + throw0(DB_ERROR("Invalid data from m_blocks")); + const uint8_t block_major_version = *((const uint8_t*)vb.mv_data); + if (block_major_version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + past_long_term_weight = true; + } + + uint64_t long_term_block_weight; + if (past_long_term_weight) + { + std::vector<uint64_t> weights(long_term_block_weights.begin(), long_term_block_weights.end()); + uint64_t long_term_effective_block_median_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, epee::misc_utils::median(weights)); + long_term_block_weight = std::min<uint64_t>(bi.bi_weight, long_term_effective_block_median_weight + long_term_effective_block_median_weight * 2 / 5); + } + else + { + long_term_block_weight = bi.bi_weight; + } + long_term_block_weights.push_back(long_term_block_weight); + bi.bi_long_term_block_weight = long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 4; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { if (oldversion < 1) @@ -4413,6 +4984,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_1_2(); if (oldversion < 3) migrate_2_3(); + if (oldversion < 4) + migrate_3_4(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 60797a076..7a04c5f0c 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -40,6 +40,11 @@ namespace cryptonote { +typedef struct txindex { + crypto::hash key; + tx_data_t data; +} txindex; + typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; @@ -53,6 +58,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txs_pruned; MDB_cursor *m_txc_txs_prunable; MDB_cursor *m_txc_txs_prunable_hash; + MDB_cursor *m_txc_txs_prunable_tip; MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_outputs; @@ -62,6 +68,8 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txpool_blob; MDB_cursor *m_txc_hf_versions; + + MDB_cursor *m_txc_properties; } mdb_txn_cursors; #define m_cur_blocks m_cursors->m_txc_blocks @@ -73,12 +81,14 @@ typedef struct mdb_txn_cursors #define m_cur_txs_pruned m_cursors->m_txc_txs_pruned #define m_cur_txs_prunable m_cursors->m_txc_txs_prunable #define m_cur_txs_prunable_hash m_cursors->m_txc_txs_prunable_hash +#define m_cur_txs_prunable_tip m_cursors->m_txc_txs_prunable_tip #define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_txpool_meta m_cursors->m_txc_txpool_meta #define m_cur_txpool_blob m_cursors->m_txc_txpool_blob #define m_cur_hf_versions m_cursors->m_txc_hf_versions +#define m_cur_properties m_cursors->m_txc_properties typedef struct mdb_rflags { @@ -92,12 +102,14 @@ typedef struct mdb_rflags bool m_rf_txs_pruned; bool m_rf_txs_prunable; bool m_rf_txs_prunable_hash; + bool m_rf_txs_prunable_tip; bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_txpool_meta; bool m_rf_txpool_blob; bool m_rf_hf_versions; + bool m_rf_properties; } mdb_rflags; typedef struct mdb_threadinfo @@ -213,13 +225,15 @@ public: virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; - virtual crypto::hash top_block_hash() const; + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const; virtual block get_top_block() const; @@ -232,6 +246,7 @@ public: virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; virtual uint64_t get_tx_count() const; @@ -242,7 +257,7 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) const; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const; virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) const; virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -252,7 +267,7 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const; - virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const; + virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const; virtual bool has_key_image(const crypto::key_image& img) const; @@ -264,6 +279,11 @@ public: virtual bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const; virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const; virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; + virtual uint32_t get_blockchain_pruning_seed() const; + virtual bool prune_blockchain(uint32_t pruning_seed = 0); + virtual bool update_pruning(); + virtual bool check_pruning(); + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, bool include_unrelayed_txes = true) const; virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; @@ -272,11 +292,12 @@ public: virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const; virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; - virtual uint64_t add_block( const block& blk + virtual uint64_t add_block( const std::pair<block, blobdata>& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated - , const std::vector<transaction>& txs + , const std::vector<std::pair<transaction, blobdata>>& txs ); virtual void set_batch_transactions(bool batch_transactions); @@ -309,6 +330,11 @@ public: bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const; + // helper functions + static int compare_uint64(const MDB_val *a, const MDB_val *b); + static int compare_hash32(const MDB_val *a, const MDB_val *b); + static int compare_string(const MDB_val *a, const MDB_val *b); + private: void do_resize(uint64_t size_increase=0); @@ -318,6 +344,7 @@ private: virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -326,7 +353,7 @@ private: virtual void remove_block(); - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); @@ -361,6 +388,8 @@ private: inline void check_open() const; + bool prune_worker(int mode, uint32_t pruning_seed); + virtual bool is_read_only() const; virtual uint64_t get_database_size() const; @@ -380,6 +409,9 @@ private: // migrate from DB version 2 to 3 void migrate_2_3(); + // migrate from DB version 3 to 4 + void migrate_3_4(); + void cleanup_batch(); private: @@ -393,6 +425,7 @@ private: MDB_dbi m_txs_pruned; MDB_dbi m_txs_prunable; MDB_dbi m_txs_prunable_hash; + MDB_dbi m_txs_prunable_tip; MDB_dbi m_tx_indices; MDB_dbi m_tx_outputs; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h new file mode 100644 index 000000000..eb9c3e45a --- /dev/null +++ b/src/blockchain_db/testdb.h @@ -0,0 +1,152 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include <string> +#include <vector> +#include <map> + +#include "blockchain_db.h" + +namespace cryptonote +{ + +class BaseTestDB: public cryptonote::BlockchainDB { +public: + BaseTestDB() {} + virtual void open(const std::string& filename, const int db_flags = 0) { } + virtual void close() {} + virtual void sync() {} + virtual void safesyncmode(const bool onoff) {} + virtual void reset() {} + virtual std::vector<std::string> get_filenames() const { return std::vector<std::string>(); } + virtual bool remove_data_file(const std::string& folder) const { return true; } + virtual std::string get_db_name() const { return std::string(); } + virtual bool lock() { return true; } + virtual void unlock() { } + virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0) { return true; } + virtual void batch_stop() {} + virtual void set_batch_transactions(bool) {} + virtual void block_txn_start(bool readonly=false) {} + virtual void block_txn_stop() {} + virtual void block_txn_abort() {} + virtual void drop_hard_fork_info() {} + virtual bool block_exists(const crypto::hash& h, uint64_t *height) const { return false; } + virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const { return cryptonote::t_serializable_object_to_blob(get_block_from_height(height)); } + virtual cryptonote::blobdata get_block_blob(const crypto::hash& h) const { return cryptonote::blobdata(); } + virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { return false; } + virtual uint64_t get_block_height(const crypto::hash& h) const { return 0; } + virtual cryptonote::block_header get_block_header(const crypto::hash& h) const { return cryptonote::block_header(); } + virtual uint64_t get_block_timestamp(const uint64_t& height) const { return 0; } + virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const { return {}; } + virtual uint64_t get_top_block_timestamp() const { return 0; } + virtual size_t get_block_weight(const uint64_t& height) const { return 128; } + virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; } + virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; } + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; } + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; } + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); } + virtual std::vector<cryptonote::block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<cryptonote::block>(); } + virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<crypto::hash>(); } + virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const { if (block_height) *block_height = 0; return crypto::hash(); } + virtual cryptonote::block get_top_block() const { return cryptonote::block(); } + virtual uint64_t height() const { return 1; } + virtual bool tx_exists(const crypto::hash& h) const { return false; } + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; } + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; } + virtual cryptonote::transaction get_tx(const crypto::hash& h) const { return cryptonote::transaction(); } + virtual bool get_tx(const crypto::hash& h, cryptonote::transaction &tx) const { return false; } + virtual uint64_t get_tx_count() const { return 0; } + virtual std::vector<cryptonote::transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const { return std::vector<cryptonote::transaction>(); } + virtual uint64_t get_tx_block_height(const crypto::hash& h) const { return 0; } + virtual uint64_t get_num_outputs(const uint64_t& amount) const { return 1; } + virtual uint64_t get_indexing_base() const { return 0; } + virtual cryptonote::output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const { return cryptonote::output_data_t(); } + virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const { return cryptonote::tx_out_index(); } + virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { return cryptonote::tx_out_index(); } + virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<cryptonote::tx_out_index> &indices) const {} + virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<cryptonote::output_data_t> &outputs, bool allow_partial = false) const {} + virtual bool can_thread_bulk_indices() const { return false; } + virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const { return std::vector<uint64_t>(); } + virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const { return std::vector<std::vector<uint64_t>>(); } + virtual bool has_key_image(const crypto::key_image& img) const { return false; } + virtual void remove_block() { } + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) {return 0;} + virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) {} + virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) {return 0;} + virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) {} + virtual void add_spent_key(const crypto::key_image& k_image) {} + virtual void remove_spent_key(const crypto::key_image& k_image) {} + + virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const { return true; } + virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const { return true; } + virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const { return true; } + virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { return true; } + virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const { return true; } + virtual bool is_read_only() const { return false; } + virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); } + virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const { return false; } + + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const cryptonote::txpool_tx_meta_t& details) {} + virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) {} + virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const { return 0; } + virtual bool txpool_has_tx(const crypto::hash &txid) const { return false; } + virtual void remove_txpool_tx(const crypto::hash& txid) {} + virtual bool get_txpool_tx_meta(const crypto::hash& txid, cryptonote::txpool_tx_meta_t &meta) const { return false; } + virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const { return false; } + virtual uint64_t get_database_size() const { return 0; } + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const { return ""; } + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = false) const { return false; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) { } + virtual cryptonote::block get_block_from_height(const uint64_t& height) const { return cryptonote::block(); } + virtual void set_hard_fork_version(uint64_t height, uint8_t version) {} + virtual uint8_t get_hard_fork_version(uint64_t height) const { return 0; } + virtual void check_hard_fork_info() {} + + virtual uint32_t get_blockchain_pruning_seed() const { return 0; } + virtual bool prune_blockchain(uint32_t pruning_seed = 0) { return true; } + virtual bool update_pruning() { return true; } + virtual bool check_pruning() { return true; } + virtual void prune_outputs(uint64_t amount) {} +}; + +} diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index ddf575c29..0ba7ee86c 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -92,6 +92,17 @@ monero_private_headers(blockchain_prune_known_spent_data +set(blockchain_prune_sources + blockchain_prune.cpp + ) + +set(blockchain_prune_private_headers) + +monero_private_headers(blockchain_prune + ${blockchain_prune_private_headers}) + + + set(blockchain_ancestry_sources blockchain_ancestry.cpp ) @@ -298,3 +309,25 @@ set_property(TARGET blockchain_prune_known_spent_data PROPERTY OUTPUT_NAME "monero-blockchain-prune-known-spent-data") install(TARGETS blockchain_prune_known_spent_data DESTINATION bin) + +monero_add_executable(blockchain_prune + ${blockchain_prune_sources} + ${blockchain_prune_private_headers}) + +set_property(TARGET blockchain_prune + PROPERTY + OUTPUT_NAME "monero-blockchain-prune") +install(TARGETS blockchain_prune DESTINATION bin) + +target_link_libraries(blockchain_prune + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md index 5d968cd75..ad5963f27 100644 --- a/src/blockchain_utilities/README.md +++ b/src/blockchain_utilities/README.md @@ -1,6 +1,6 @@ # Monero Blockchain Utilities -Copyright (c) 2014-2018, The Monero Project +Copyright (c) 2014-2019, The Monero Project ## Introduction diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index e01a8892c..a6ee0573f 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -51,6 +51,8 @@ using namespace epee; using namespace cryptonote; static bool stop_requested = false; +static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; +static bool opt_cache_outputs = false, opt_cache_txes = false, opt_cache_blocks = false; struct ancestor { @@ -137,6 +139,8 @@ struct ancestry_state_t std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; std::vector<cryptonote::block> block_cache; + ancestry_state_t(): height(0) {} + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) { a & height; @@ -219,6 +223,113 @@ static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto return i->second; } +static bool get_block_from_height(ancestry_state_t &state, BlockchainDB *db, uint64_t height, cryptonote::block &b) +{ + ++total_blocks; + if (state.block_cache.size() > height && !state.block_cache[height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[height]; + return true; + } + cryptonote::blobdata bd = db->get_block_blob_from_height(height); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return false; + } + if (opt_cache_blocks) + { + state.block_cache.resize(height + 1); + state.block_cache[height] = b; + } + return true; +} + +static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const crypto::hash &txid, ::tx_data_t &tx_data) +{ + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + return true; + } + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return false; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return false; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + return true; +} + +static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t amount, uint64_t offset, crypto::hash &txid) +{ + ++total_outputs; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + if (i != state.output_cache.end()) + { + ++cached_outputs; + txid = i->second; + return true; + } + + const output_data_t od = db->get_output_key(amount, offset, false); + cryptonote::block b; + if (!get_block_from_height(state, db, od.height, b)) + return false; + + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + txid = cryptonote::get_transaction_hash(b.miner_tx); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return false; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + ::tx_data_t tx_data3; + if (!get_transaction(state, db, block_txid, tx_data3)) + return false; + + for (size_t out = 0; out < tx_data3.vout.size(); ++out) + { + if (tx_data3.vout[out] == od.pubkey) + { + txid = block_txid; + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + } + return false; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -243,12 +354,13 @@ int main(int argc, char* argv[]) "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<std::string> arg_output = {"output", "Get ancestry for this output (amount/offset format)", ""}; const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; - const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_refresh = {"refresh", "Refresh the whole chain first", false}; const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; - const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx in per height average", false}; const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); @@ -257,8 +369,9 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_output); command_line::add_arg(desc_cmd_sett, arg_height); - command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_refresh); command_line::add_arg(desc_cmd_sett, arg_cache_outputs); command_line::add_arg(desc_cmd_sett, arg_cache_txes); command_line::add_arg(desc_cmd_sett, arg_cache_blocks); @@ -300,20 +413,22 @@ int main(int argc, char* argv[]) bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + std::string opt_output_string = command_line::get_arg(vm, arg_output); uint64_t opt_height = command_line::get_arg(vm, arg_height); - bool opt_all = command_line::get_arg(vm, arg_all); - bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); - bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); - bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_refresh = command_line::get_arg(vm, arg_refresh); + opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); - if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1) { - std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + std::cerr << "Only one of --txid, --height, --output can be given" << std::endl; return 1; } crypto::hash opt_txid = crypto::null_hash; + uint64_t output_amount = 0, output_offset = 0; if (!opt_txid_string.empty()) { if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) @@ -322,6 +437,14 @@ int main(int argc, char* argv[]) return 1; } } + else if (!opt_output_string.empty()) + { + if (sscanf(opt_output_string.c_str(), "%" SCNu64 "/%" SCNu64, &output_amount, &output_offset) != 2) + { + std::cerr << "Invalid output" << std::endl; + return 1; + } + } std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -372,37 +495,36 @@ int main(int argc, char* argv[]) std::vector<crypto::hash> start_txids; - // forward method - if (opt_all) - { - uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; - ancestry_state_t state; + ancestry_state_t state; - const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try { - try - { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; - } - catch (const std::exception &e) - { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = ancestry_state_t(); - } - state_data_in.close(); + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); } + state_data_in.close(); + } - tools::signal_handler::install([](int type) { - stop_requested = true; - }); + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + // forward method + const uint64_t db_height = db->height(); + if (opt_refresh) + { MINFO("Starting from height " << state.height); - const uint64_t db_height = db->height(); state.block_cache.reserve(db_height); for (uint64_t h = state.height; h < db_height; ++h) { @@ -464,113 +586,20 @@ int main(int argc, char* argv[]) { for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) { - if (1) + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) { - const uint64_t amount = tx_data.vin[ring].first; - const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; - for (uint64_t offset: absolute_offsets) + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + // find the tx which created this output + bool found = false; + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) { - const output_data_t od = db->get_output_key(amount, offset); - add_ancestry(state.ancestry, txid, ancestor{amount, offset}); - cryptonote::block b; - ++total_blocks; - if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) - { - ++cached_blocks; - b = state.block_cache[od.height]; - } - else - { - cryptonote::blobdata bd = db->get_block_blob_from_height(od.height); - if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) - { - LOG_PRINT_L0("Bad block from db"); - return 1; - } - if (opt_cache_blocks) - { - state.block_cache.resize(od.height + 1); - state.block_cache[od.height] = b; - } - } - // find the tx which created this output - bool found = false; - std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); - ++total_outputs; - if (i != state.output_cache.end()) - { - ++cached_outputs; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); - found = true; - } - else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) - { - if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); - if (opt_cache_outputs) - state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); - return 1; - } - } - for (const crypto::hash &block_txid: b.tx_hashes) - { - if (found) - break; - ::tx_data_t tx_data2; - std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); - ++total_txes; - if (i != state.tx_cache.end()) - { - ++cached_txes; - tx_data2 = i->second; - } - else - { - cryptonote::blobdata bd; - if (!db->get_pruned_tx_blob(block_txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); - return 1; - } - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) - { - LOG_PRINT_L0("Bad tx: " << block_txid); - return 1; - } - tx_data2 = ::tx_data_t(tx); - if (opt_cache_txes) - state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); - } - for (size_t out = 0; out < tx_data2.vout.size(); ++out) - { - if (tx_data2.vout[out] == od.pubkey) - { - found = true; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); - if (opt_cache_outputs) - state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); - break; - } - } - } - if (!found) - { - LOG_PRINT_L0("Output originating transaction not found"); - return 1; - } + LOG_PRINT_L0("Output originating transaction not found"); + return 1; } + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); } } } @@ -581,10 +610,6 @@ int main(int argc, char* argv[]) if (!txids.empty()) { std::string stats_msg; - if (opt_show_cache_stats) - stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) - + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " - + std::to_string(cached_outputs*100./total_outputs); MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); } state.height = h; @@ -608,14 +633,30 @@ int main(int argc, char* argv[]) } state_data_out.close(); } - - goto done; + } + else + { + if (state.height < db_height) + { + MWARNING("The state file is only built up to height " << state.height << ", but the blockchain reached height " << db_height); + MWARNING("You may want to run with --refresh if you want to get ancestry for newer data"); + } } if (!opt_txid_string.empty()) { start_txids.push_back(opt_txid); } + else if (!opt_output_string.empty()) + { + crypto::hash txid; + if (!get_output_txid(state, db, output_amount, output_offset, txid)) + { + LOG_PRINT_L0("Output not found in db"); + return 1; + } + start_txids.push_back(txid); + } else { const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); @@ -648,108 +689,40 @@ int main(int argc, char* argv[]) const crypto::hash txid = txids.front(); txids.pop_front(); - cryptonote::blobdata bd; - if (!db->get_pruned_tx_blob(txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << txid << " from db"); - return 1; - } - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) - { - LOG_PRINT_L0("Bad tx: " << txid); + if (stop_requested) + goto done; + + ::tx_data_t tx_data2; + if (!get_transaction(state, db, txid, tx_data2)) return 1; - } - const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + + const bool coinbase = tx_data2.coinbase; if (coinbase) continue; - for (size_t ring = 0; ring < tx.vin.size(); ++ring) + for (size_t ring = 0; ring < tx_data2.vin.size(); ++ring) { - if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) { - const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); - const uint64_t amount = txin.amount; - auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + const uint64_t amount = tx_data2.vin[ring].first; + auto absolute_offsets = tx_data2.vin[ring].second; for (uint64_t offset: absolute_offsets) { add_ancestor(ancestry, amount, offset); - const output_data_t od = db->get_output_key(amount, offset); - bd = db->get_block_blob_from_height(od.height); - cryptonote::block b; - if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) - { - LOG_PRINT_L0("Bad block from db"); - return 1; - } + // find the tx which created this output bool found = false; - for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) - { - if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); - MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); - return 1; - } - } - for (const crypto::hash &block_txid: b.tx_hashes) - { - if (found) - break; - if (!db->get_pruned_tx_blob(block_txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); - return 1; - } - cryptonote::transaction tx2; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) - { - LOG_PRINT_L0("Bad tx: " << block_txid); - return 1; - } - for (size_t out = 0; out < tx2.vout.size(); ++out) - { - if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - txids.push_back(block_txid); - MDEBUG("adding txid: " << block_txid); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << block_txid); - return 1; - } - } - } - if (!found) + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) { LOG_PRINT_L0("Output originating transaction not found"); return 1; } + + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); + txids.push_back(output_txid); + MDEBUG("adding txid: " << output_txid); } } - else - { - LOG_PRINT_L0("Bad vin type in txid " << txid); - return 1; - } } } @@ -762,6 +735,13 @@ int main(int argc, char* argv[]) done: core_storage->deinit(); + + if (opt_show_cache_stats) + MINFO("cache: txes " << std::to_string(cached_txes*100./total_txes) + << "%, blocks " << std::to_string(cached_blocks*100./total_blocks) + << "%, outputs " << std::to_string(cached_outputs*100./total_outputs) + << "%"); + return 0; CATCH_ENTRY("Depth query error", 1); diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 73819bd25..6ff184041 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -1131,7 +1131,7 @@ int main(int argc, char* argv[]) return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-find-spent-outputs.log"), true); + mlog_configure(mlog_get_default_log_path("monero-blockchain-mark-spent-outputs.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index 8060b0de4..8be83ee67 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index 5a49f3478..fa1243c1f 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -177,6 +177,12 @@ int main(int argc, char* argv[]) } r = core_storage->init(db, opt_testnet ? cryptonote::TESTNET : opt_stagenet ? cryptonote::STAGENET : cryptonote::MAINNET); + if (core_storage->get_blockchain_pruning_seed()) + { + LOG_PRINT_L0("Blockchain is pruned, cannot export"); + return 1; + } + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); LOG_PRINT_L0("Source blockchain storage initialized OK"); LOG_PRINT_L0("Exporting blockchain raw data..."); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 2a8a6f494..e4efdc3cb 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -193,8 +193,16 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block } core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes); - core.prepare_handle_incoming_blocks(blocks); + std::vector<block> pblocks; + core.prepare_handle_incoming_blocks(blocks, pblocks); + if (!pblocks.empty() && pblocks.size() != blocks.size()) + { + MERROR("Unexpected parsed blocks size"); + core.cleanup_handle_incoming_blocks(); + return 1; + } + size_t blockidx = 0; for(const block_complete_entry& block_entry: blocks) { // process transactions @@ -215,7 +223,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block block_verification_context bvc = boost::value_initialized<block_verification_context>(); - core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block + core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, false); // <--- process block if(bvc.m_verifivation_failed) { @@ -455,7 +463,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path } else { - std::vector<transaction> txs; + std::vector<std::pair<transaction, blobdata>> txs; std::vector<transaction> archived_txs; archived_txs = bp.txs; @@ -472,7 +480,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path // because add_block() calls // add_transaction(blk_hash, blk.miner_tx) first, and // then a for loop for the transactions in txs. - txs.push_back(tx); + txs.push_back(std::make_pair(tx, tx_to_blob(tx))); } size_t block_weight; @@ -485,7 +493,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path try { - core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs); + uint64_t long_term_block_weight = core.get_blockchain_storage().get_next_long_term_block_weight(block_weight); + core.get_blockchain_storage().get_db().add_block(std::make_pair(b, block_to_blob(b)), block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp new file mode 100644 index 000000000..36080aade --- /dev/null +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -0,0 +1,681 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <array> +#include <lmdb.h> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/pruning.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static std::string db_path; + +// default to fast:1 +static uint64_t records_per_sync = 128; +static const size_t slack = 512 * 1024 * 1024; + +static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name) +{ + std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string()); + if (ec) + MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message()); + return ec; +} + +static void open(MDB_env *&env, const boost::filesystem::path &path, uint64_t db_flags, bool readonly) +{ + int dbr; + int flags = 0; + + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + if (readonly) + flags |= MDB_RDONLY; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 32); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_open(env, path.string().c_str(), flags, 0664); + if (dbr) throw std::runtime_error("Failed to open database file '" + + path.string() + "': " + std::string(mdb_strerror(dbr))); +} + +static void close(MDB_env *env) +{ + mdb_env_close(env); +} + +static void add_size(MDB_env *env, uint64_t bytes) +{ + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < bytes) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << + (si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed"); + return; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + MDB_envinfo mei; + mdb_env_info(env, &mei); + MDB_stat mst; + mdb_env_stat(env, &mst); + + uint64_t new_mapsize = (uint64_t)mei.me_mapsize + bytes; + new_mapsize += (new_mapsize % mst.ms_psize); + + int result = mdb_env_set_mapsize(env, new_mapsize); + if (result) + throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result))); + + MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); +} + +static void check_resize(MDB_env *env, size_t bytes) +{ + MDB_envinfo mei; + MDB_stat mst; + + mdb_env_info(env, &mei); + mdb_env_stat(env, &mst); + + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + if (size_used + bytes + slack >= mei.me_mapsize) + add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize); +} + +static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) +{ + if (nrecords % records_per_sync && bytes <= slack / 2) + return false; + int dbr = mdb_txn_commit(*txn); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + check_resize(env, bytes); + dbr = mdb_txn_begin(env, NULL, 0, txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + bytes = 0; + return true; +} + +static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0) +{ + MDB_dbi dbi0, dbi1; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0, *cur1; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MINFO("Copying " << table); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, table, flags, &dbi0); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp); + + dbr = mdb_dbi_open(txn1, table, flags, &dbi1); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp); + + dbr = mdb_txn_commit(txn1); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active1 = false; + MDB_stat stats; + dbr = mdb_env_stat(env0, &stats); + if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize); + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_drop(txn1, dbi1, 0); + if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_cursor_open(txn0, dbi0, &cur0); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_val k; + MDB_val v; + MDB_cursor_op op = MDB_FIRST; + size_t nrecords = 0, bytes = 0; + while (1) + { + int ret = mdb_cursor_get(cur0, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret))); + + bytes += k.mv_size + v.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + + ret = mdb_cursor_put(cur1, &k, &v, putflags); + if (ret) + throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret))); + } + + mdb_cursor_close(cur1); + mdb_cursor_close(cur0); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1); + mdb_dbi_close(env0, dbi0); +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret))); + if (v.mv_size == 0) + throw std::runtime_error("Invalid transaction pruned data"); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +static void prune(MDB_env *env0, MDB_env *env1) +{ + MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MGINFO("Creating pruned txs_prunable"); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32); + dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_drop(txn1, dbi1_txs_prunable, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + MDB_val k, v; + uint32_t pruning_seed = tools::make_pruning_seed(tools::get_random_stripe(), CRYPTONOTE_PRUNING_LOG_STRIPES); + static char pruning_seed_key[] = "pruning_seed"; + k.mv_data = pruning_seed_key; + k.mv_size = strlen("pruning_seed") + 1; + v.mv_data = (void*)&pruning_seed; + v.mv_size = sizeof(pruning_seed); + dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0); + if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr))); + + MDB_stat stats; + dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_stat(txn0, dbi0_blocks, &stats); + if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr))); + mdb_dbi_close(env0, dbi0_blocks); + const uint64_t blockchain_height = stats.ms_entries; + size_t nrecords = 0, bytes = 0; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret))); + + const txindex *ti = (const txindex*)v.mv_data; + const uint64_t block_height = ti->data.block_id; + MDB_val_set(kk, ti->data.tx_id); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDEBUG(block_height << "/" << blockchain_height << " is in tip"); + MDB_val_set(vv, block_height); + dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + } + if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk)) + { + MDB_val vv; + dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET); + if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + } + else + { + MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping"); + } + } + + mdb_cursor_close(cur1_txs_prunable_tip); + mdb_cursor_close(cur1_txs_prunable); + mdb_cursor_close(cur0_txs_prunable); + mdb_cursor_close(cur0_txs_pruned); + mdb_cursor_close(cur0_tx_indices); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1_properties); + mdb_dbi_close(env1, dbi1_txs_prunable_tip); + mdb_dbi_close(env1, dbi1_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_pruned); + mdb_dbi_close(env0, dbi0_tx_indices); +} + +static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags) +{ + std::vector<std::string> options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + db_flags = 0; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) + { + if(options[0] == "safe") + { + safemode = true; + db_flags = DBF_SAFE; + } + else if(options[0] == "fast") + { + db_flags = DBF_FAST; + } + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + return false; + } + + if(options.size() >= 2 && !safemode) + { + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr != '\0') + return false; + records_per_sync = bps; + } + + return true; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-prune.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + MINFO("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database); + std::string data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\")) + data_dir.pop_back(); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + MERROR("Invalid database type: " << db_type); + return 1; + } + if (db_type != "lmdb") + { + MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported"); + return 1; + } + + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + uint64_t db_flags = 0; + if (!parse_db_sync_mode(db_sync_mode, db_flags)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + MINFO("Initializing source blockchain (BlockchainDB)"); + std::array<std::unique_ptr<Blockchain>, 2> core_storage; + Blockchain *blockchain = NULL; + tx_memory_pool m_mempool(*blockchain); + boost::filesystem::path paths[2]; + bool already_pruned = false; + for (size_t n = 0; n < core_storage.size(); ++n) + { + core_storage[n].reset(new Blockchain(m_mempool)); + + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + MERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + MDEBUG("database: " << db_type); + + if (n == 1) + { + paths[1] = boost::filesystem::path(data_dir) / (db->get_db_name() + "-pruned"); + if (boost::filesystem::exists(paths[1])) + { + if (!boost::filesystem::is_directory(paths[1])) + { + MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string()); + return 1; + } + } + else + { + if (!boost::filesystem::create_directories(paths[1])) + { + MERROR("Failed to create directory: " << paths[1].string()); + return 1; + } + } + db_path = paths[1].string(); + } + else + { + paths[0] = boost::filesystem::path(data_dir) / db->get_db_name(); + } + + MINFO("Loading blockchain from folder " << paths[n] << " ..."); + + try + { + db->open(paths[n].string(), n == 0 ? DBF_RDONLY : 0); + } + catch (const std::exception& e) + { + MERROR("Error opening database: " << e.what()); + return 1; + } + r = core_storage[n]->init(db, net_type); + + std::string source_dest = n == 0 ? "source" : "pruned"; + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage"); + MINFO(source_dest << " blockchain storage initialized OK"); + if (n == 0 && core_storage[0]->get_blockchain_pruning_seed()) + { + if (!opt_copy_pruned_database) + { + MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway"); + return 1; + } + already_pruned = true; + } + if (n == 0) + { + const uint64_t blockchain_height = core_storage[0]->get_current_blockchain_height(); + const crypto::hash hash = core_storage[0]->get_block_id_by_height(blockchain_height - 1); + cryptonote::block block; + if (core_storage[0]->get_block_by_hash(hash, block)) + { + if (block.major_version < 10) + { + time_t now = time(NULL); + if (now < 1555286400) // 15 april 2019 + { + MERROR("Pruning before v10 will confuse peers. Wait for v10 first"); + return 1; + } + } + } + } + } + core_storage[0]->deinit(); + core_storage[0].reset(NULL); + core_storage[1]->deinit(); + core_storage[1].reset(NULL); + + MINFO("Pruning..."); + MDB_env *env0 = NULL, *env1 = NULL; + open(env0, paths[0], db_flags, true); + open(env1, paths[1], db_flags, false); + copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + //copy_table(env0, env1, "txs", MDB_INTEGERKEY); + copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND); + // not copied: prunable, prunable_tip + copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string); + if (already_pruned) + { + copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64); + } + else + { + prune(env0, env1); + } + close(env1); + close(env0); + + MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired"); + if (replace_file(paths[0].string(), paths[0].string() + "-old") || replace_file(paths[1].string(), paths[0].string())) + { + MERROR("Blockchain pruned OK, but renaming failed"); + return 1; + } + + MINFO("Blockchain pruned OK"); + return 0; + + CATCH_ENTRY("Pruning error", 1); +} diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index f6136c1ba..2d49b6ecd 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index aae8f333b..4cc84bf4a 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 38a0b2648..bd73350b3 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blockchain_utilities.h b/src/blockchain_utilities/blockchain_utilities.h index e690305c4..78487b995 100644 --- a/src/blockchain_utilities/blockchain_utilities.h +++ b/src/blockchain_utilities/blockchain_utilities.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp index 45ef33acb..f56ff5f94 100644 --- a/src/blockchain_utilities/blocksdat_file.cpp +++ b/src/blockchain_utilities/blocksdat_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h index 70a1f30a7..315713424 100644 --- a/src/blockchain_utilities/blocksdat_file.h +++ b/src/blockchain_utilities/blocksdat_file.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index a8c46d661..fb9a24f5d 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index 187db0938..5fb2cf366 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 278a7b40f..554c6d56e 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/blocks/CMakeLists.txt b/src/blocks/CMakeLists.txt index ff48af6dc..0b7b02fee 100644 --- a/src/blocks/CMakeLists.txt +++ b/src/blocks/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat Binary files differindex 03e2cd2a8..adc433522 100644 --- a/src/blocks/checkpoints.dat +++ b/src/blocks/checkpoints.dat diff --git a/src/checkpoints/CMakeLists.txt b/src/checkpoints/CMakeLists.txt index 715006522..d324f518e 100644 --- a/src/checkpoints/CMakeLists.txt +++ b/src/checkpoints/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index 96f575b2d..e31b96646 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -208,6 +208,7 @@ namespace cryptonote ADD_CHECKPOINT(1530000, "01759bce497ec38e63c78b1038892169203bb78f87e488172f6b854fcd63ba7e"); ADD_CHECKPOINT(1579000, "7d0d7a2346373afd41ed1e744a939fc5d474a7dbaa257be5c6fff4009e789241"); ADD_CHECKPOINT(1668900, "ac2dcaf3d2f58ffcf8391639f0f1ebafcb8eac43c49479c7c37f611868d07568"); + ADD_CHECKPOINT(1775600, "1c6e01c661dc22cab939e79ec6a5272190624ce8356d2f7b958e4f9a57fdb05e"); return true; } diff --git a/src/checkpoints/checkpoints.h b/src/checkpoints/checkpoints.h index ad2b44d1a..a55b94bf0 100644 --- a/src/checkpoints/checkpoints.h +++ b/src/checkpoints/checkpoints.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 3b1eb6d23..f06737b31 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -40,10 +40,12 @@ set(common_sources notify.cpp password.cpp perf_timer.cpp + pruning.cpp spawn.cpp threadpool.cpp updates.cpp aligned.c + timings.cc combinator.cpp) if (STACK_TRACE) @@ -69,6 +71,7 @@ set(common_private_headers http_connection.h notify.h pod-class.h + pruning.h rpc_client.h scoped_message_writer.h unordered_containers_boost_serialization.h @@ -82,6 +85,7 @@ set(common_private_headers threadpool.h updates.h aligned.h + timings.h combinator.h) monero_private_headers(common diff --git a/src/common/aligned.c b/src/common/aligned.c index 763dfd0e7..6982409f7 100644 --- a/src/common/aligned.c +++ b/src/common/aligned.c @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/apply_permutation.h b/src/common/apply_permutation.h index ff346bab1..a4b2c8b78 100644 --- a/src/common/apply_permutation.h +++ b/src/common/apply_permutation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 3562af486..ac1bf4b29 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/base58.h b/src/common/base58.h index 69611859d..6bf2c3bb7 100644 --- a/src/common/base58.h +++ b/src/common/base58.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/boost_serialization_helper.h b/src/common/boost_serialization_helper.h index 3f5c623f8..2280f3312 100644 --- a/src/common/boost_serialization_helper.h +++ b/src/common/boost_serialization_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 35135ea18..cae744ea5 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/command_line.h b/src/common/command_line.h index 3a869bb26..b5e3d94a7 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/common_fwd.h b/src/common/common_fwd.h index 2924d9cbe..7eaa6cdee 100644 --- a/src/common/common_fwd.h +++ b/src/common/common_fwd.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/compat/glibc_compat.cpp b/src/common/compat/glibc_compat.cpp index bf567987d..435e7930f 100644 --- a/src/common/compat/glibc_compat.cpp +++ b/src/common/compat/glibc_compat.cpp @@ -82,7 +82,7 @@ __explicit_bzero_chk (void *dst, size_t len, size_t dstlen) #undef glob extern "C" int glob_old(const char * pattern, int flags, int (*errfunc) (const char *epath, int eerrno), glob_t *pglob); #ifdef __i386__ -__asm__(".symver glob_old,glob@GLIBC_2.1"); +__asm__(".symver glob_old,glob@GLIBC_2.0"); #elif defined(__amd64__) __asm__(".symver glob_old,glob@GLIBC_2.2.5"); #elif defined(__arm__) diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 417b5b4ac..711a8ba30 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -232,13 +232,24 @@ public: char *str; }; +static void add_anchors(ub_ctx *ctx) +{ + const char * const *ds = ::get_builtin_ds(); + while (*ds) + { + MINFO("adding trust anchor: " << *ds); + ub_ctx_add_ta(ctx, string_copy(*ds++)); + } +} + DNSResolver::DNSResolver() : m_data(new DNSResolverData()) { int use_dns_public = 0; std::vector<std::string> dns_public_addr; - if (auto res = getenv("DNS_PUBLIC")) + const char *DNS_PUBLIC = getenv("DNS_PUBLIC"); + if (DNS_PUBLIC) { - dns_public_addr = tools::dns_utils::parse_dns_public(res); + dns_public_addr = tools::dns_utils::parse_dns_public(DNS_PUBLIC); if (!dns_public_addr.empty()) { MGINFO("Using public DNS server(s): " << boost::join(dns_public_addr, ", ") << " (TCP)"); @@ -266,11 +277,28 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - const char * const *ds = ::get_builtin_ds(); - while (*ds) + add_anchors(m_data->m_ub_context); + + if (!DNS_PUBLIC) { - MINFO("adding trust anchor: " << *ds); - ub_ctx_add_ta(m_data->m_ub_context, string_copy(*ds++)); + // if no DNS_PUBLIC specified, we try a lookup to what we know + // should be a valid DNSSEC record, and switch to known good + // DNSSEC resolvers if verification fails + bool available, valid; + static const char *probe_hostname = "updates.moneropulse.org"; + auto records = get_txt_record(probe_hostname, available, valid); + if (!valid) + { + MINFO("Failed to verify DNSSEC record from " << probe_hostname << ", falling back to TCP with well known DNSSEC resolvers"); + ub_ctx_delete(m_data->m_ub_context); + m_data->m_ub_context = ub_ctx_create(); + add_anchors(m_data->m_ub_context); + dns_public_addr = tools::dns_utils::parse_dns_public(DNS_PUBLIC); + for (const auto &ip: dns_public_addr) + ub_ctx_set_fwd(m_data->m_ub_context, string_copy(ip.c_str())); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-udp:"), string_copy("no")); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-tcp:"), string_copy("yes")); + } } } @@ -514,12 +542,12 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std if (!avail[cur_index]) { records[cur_index].clear(); - LOG_PRINT_L2("DNSSEC not available for checkpoint update at URL: " << url << ", skipping."); + LOG_PRINT_L2("DNSSEC not available for hostname: " << url << ", skipping."); } if (!valid[cur_index]) { records[cur_index].clear(); - LOG_PRINT_L2("DNSSEC validation failed for checkpoint update at URL: " << url << ", skipping."); + LOG_PRINT_L2("DNSSEC validation failed for hostname: " << url << ", skipping."); } cur_index++; @@ -541,7 +569,7 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std if (num_valid_records < 2) { - LOG_PRINT_L0("WARNING: no two valid MoneroPulse DNS checkpoint records were received"); + LOG_PRINT_L0("WARNING: no two valid DNS TXT records were received"); return false; } @@ -563,7 +591,7 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std if (good_records_index < 0) { - LOG_PRINT_L0("WARNING: no two MoneroPulse DNS checkpoint records matched"); + LOG_PRINT_L0("WARNING: no two DNS TXT records matched"); return false; } diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index 3a6ef68a1..a6bc7463a 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/download.cpp b/src/common/download.cpp index 58ce0595f..f07d6798d 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -179,8 +179,8 @@ namespace tools lock.unlock(); - bool ssl = u_c.schema == "https"; - uint16_t port = u_c.port ? u_c.port : ssl ? 443 : 80; + epee::net_utils::ssl_support_t ssl = u_c.schema == "https" ? epee::net_utils::ssl_support_t::e_ssl_support_enabled : epee::net_utils::ssl_support_t::e_ssl_support_disabled; + uint16_t port = u_c.port ? u_c.port : ssl == epee::net_utils::ssl_support_t::e_ssl_support_enabled ? 443 : 80; MDEBUG("Connecting to " << u_c.host << ":" << port); client.set_server(u_c.host, std::to_string(port), boost::none, ssl); if (!client.connect(std::chrono::seconds(30))) diff --git a/src/common/download.h b/src/common/download.h index 3097394bc..f8656a59c 100644 --- a/src/common/download.h +++ b/src/common/download.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/http_connection.h b/src/common/http_connection.h index 554dd832b..6b4294802 100644 --- a/src/common/http_connection.h +++ b/src/common/http_connection.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index ffe8d8b52..9ac347263 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -38,6 +38,8 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" +#define MAX_LANGUAGE_SIZE 16 + static const unsigned char qm_magic[16] = {0x3c, 0xb8, 0x64, 0x18, 0xca, 0xef, 0x9c, 0x95, 0xcd, 0x21, 0x1c, 0xbf, 0x60, 0xa1, 0xbd, 0xdd}; static std::map<std::string,std::string> i18n_entries; @@ -62,7 +64,19 @@ std::string i18n_get_language() std::string language = e; language = language.substr(0, language.find(".")); + language = language.substr(0, language.find("@")); + + // check valid values + for (char c: language) + if (!strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-.@", c)) + return "en"; + std::transform(language.begin(), language.end(), language.begin(), tolower); + if (language.size() > MAX_LANGUAGE_SIZE) + { + i18n_log("Language from LANG/LC_ALL suspiciously long, defaulting to en"); + return "en"; + } return language; } diff --git a/src/common/i18n.h b/src/common/i18n.h index d21d00275..82a07410d 100644 --- a/src/common/i18n.h +++ b/src/common/i18n.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/json_util.h b/src/common/json_util.h index c320c3956..96f4b90e6 100644 --- a/src/common/json_util.h +++ b/src/common/json_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/notify.cpp b/src/common/notify.cpp index cadc68ea7..e2df5096d 100644 --- a/src/common/notify.cpp +++ b/src/common/notify.cpp @@ -27,11 +27,15 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <stdarg.h> #include "misc_log_ex.h" #include "file_io_utils.h" #include "spawn.h" #include "notify.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "notify" + namespace tools { @@ -44,17 +48,34 @@ Notify::Notify(const char *spec) { CHECK_AND_ASSERT_THROW_MES(spec, "Null spec"); - boost::split(args, spec, boost::is_any_of(" ")); + boost::split(args, spec, boost::is_any_of(" \t"), boost::token_compress_on); CHECK_AND_ASSERT_THROW_MES(args.size() > 0, "Failed to parse spec"); + if (strchr(spec, '\'') || strchr(spec, '\"') || strchr(spec, '\\')) + MWARNING("A notification spec contains a quote or backslash: note that these are handled verbatim, which may not be the intent"); filename = args[0]; CHECK_AND_ASSERT_THROW_MES(epee::file_io_utils::is_file_exist(filename), "File not found: " << filename); } -int Notify::notify(const char *parameter) +static void replace(std::vector<std::string> &v, const char *tag, const char *s) +{ + for (std::string &str: v) + boost::replace_all(str, tag, s); +} + +int Notify::notify(const char *tag, const char *s, ...) { std::vector<std::string> margs = args; - for (std::string &s: margs) - boost::replace_all(s, "%s", parameter); + + replace(margs, tag, s); + + va_list ap; + va_start(ap, s); + while ((tag = va_arg(ap, const char*))) + { + s = va_arg(ap, const char*); + replace(margs, tag, s); + } + va_end(ap); return tools::spawn(filename.c_str(), margs, false); } diff --git a/src/common/notify.h b/src/common/notify.h index 81aacebb0..f813e8def 100644 --- a/src/common/notify.h +++ b/src/common/notify.h @@ -39,7 +39,7 @@ class Notify public: Notify(const char *spec); - int notify(const char *parameter); + int notify(const char *tag, const char *s, ...); private: std::string filename; diff --git a/src/common/password.cpp b/src/common/password.cpp index 5f5cb800a..03d13db42 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/password.h b/src/common/password.h index beb98283b..2837c70f3 100644 --- a/src/common/password.h +++ b/src/common/password.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 3e1357833..dda498088 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index d859cf576..717391623 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -53,6 +53,7 @@ public: void resume(); void reset(); uint64_t value() const; + operator uint64_t() const { return value(); } protected: uint64_t ticks; diff --git a/src/common/pod-class.h b/src/common/pod-class.h index 5f6709eef..200647590 100644 --- a/src/common/pod-class.h +++ b/src/common/pod-class.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/pruning.cpp b/src/common/pruning.cpp new file mode 100644 index 000000000..442b24e4e --- /dev/null +++ b/src/common/pruning.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_config.h" +#include "misc_log_ex.h" +#include "crypto/crypto.h" +#include "pruning.h" + +namespace tools +{ + +uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes) +{ + CHECK_AND_ASSERT_THROW_MES(log_stripes <= PRUNING_SEED_LOG_STRIPES_MASK, "log_stripes out of range"); + CHECK_AND_ASSERT_THROW_MES(stripe > 0 && stripe <= (1ul << log_stripes), "stripe out of range"); + return (log_stripes << PRUNING_SEED_LOG_STRIPES_SHIFT) | ((stripe - 1) << PRUNING_SEED_STRIPE_SHIFT); +} + +bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return true; + const uint32_t log_stripes = get_pruning_log_stripes(pruning_seed); + uint32_t block_stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + return block_stripe == 0 || block_stripe == stripe; +} + +uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return 0; + return ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & (uint64_t)((1ul << log_stripes) - 1)) + 1; +} + +uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + const uint32_t stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + if (stripe == 0) + return 0; + return make_pruning_seed(stripe, log_stripes); +} + +uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + CHECK_AND_ASSERT_MES(block_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "block_height too large"); + CHECK_AND_ASSERT_MES(blockchain_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "blockchain_height too large"); + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return block_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return block_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_stripe = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_stripe == stripe) + return block_height; + const uint64_t cycles = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) >> log_stripes); + const uint64_t cycle_start = cycles + ((stripe > block_pruning_stripe) ? 0 : 1); + const uint64_t h = cycle_start * (CRYPTONOTE_PRUNING_STRIPE_SIZE << log_stripes) + (stripe - 1) * CRYPTONOTE_PRUNING_STRIPE_SIZE; + if (h + CRYPTONOTE_PRUNING_TIP_BLOCKS > blockchain_height) + return blockchain_height < CRYPTONOTE_PRUNING_TIP_BLOCKS ? 0 : blockchain_height - CRYPTONOTE_PRUNING_TIP_BLOCKS; + CHECK_AND_ASSERT_MES(h >= block_height, block_height, "h < block_height, unexpected"); + return h; +} + +uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return blockchain_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return blockchain_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_seed = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_seed != stripe) + return block_height; + const uint32_t next_stripe = 1 + (block_pruning_seed & mask); + return get_next_unpruned_block_height(block_height, blockchain_height, tools::make_pruning_seed(next_stripe, log_stripes)); +} + +uint32_t get_random_stripe() +{ + return 1 + crypto::rand<uint8_t>() % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES); +} + +} + diff --git a/src/common/pruning.h b/src/common/pruning.h new file mode 100644 index 000000000..3fac3c0fa --- /dev/null +++ b/src/common/pruning.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stdint.h> + +namespace tools +{ + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_SHIFT = 7; + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_MASK = 0x7; + static constexpr uint32_t PRUNING_SEED_STRIPE_SHIFT = 0; + static constexpr uint32_t PRUNING_SEED_STRIPE_MASK = 0x7f; + + constexpr inline uint32_t get_pruning_log_stripes(uint32_t pruning_seed) { return (pruning_seed >> PRUNING_SEED_LOG_STRIPES_SHIFT) & PRUNING_SEED_LOG_STRIPES_MASK; } + inline uint32_t get_pruning_stripe(uint32_t pruning_seed) { if (pruning_seed == 0) return 0; return 1 + ((pruning_seed >> PRUNING_SEED_STRIPE_SHIFT) & PRUNING_SEED_STRIPE_MASK); } + + uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes); + + bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_random_stripe(); +} + diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h index 9665966ae..cb5f79da8 100644 --- a/src/common/rpc_client.h +++ b/src/common/rpc_client.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index 42f439ad8..546377392 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/sfinae_helpers.h b/src/common/sfinae_helpers.h index fa5052a2e..e9a98bb63 100644 --- a/src/common/sfinae_helpers.h +++ b/src/common/sfinae_helpers.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/spawn.cpp b/src/common/spawn.cpp index b2d03f62f..9a7e75d41 100644 --- a/src/common/spawn.cpp +++ b/src/common/spawn.cpp @@ -42,6 +42,9 @@ #include "util.h" #include "spawn.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "spawn" + namespace tools { @@ -88,7 +91,7 @@ int spawn(const char *filename, const std::vector<std::string>& args, bool wait) MINFO("Child exited with " << exitCode); return static_cast<int>(exitCode); #else - char **argv = (char**)alloca(sizeof(char*) * (args.size() + 1)); + std::vector<char*> argv(args.size() + 1); for (size_t n = 0; n < args.size(); ++n) argv[n] = (char*)args[n].c_str(); argv[args.size()] = NULL; @@ -106,7 +109,7 @@ int spawn(const char *filename, const std::vector<std::string>& args, bool wait) tools::closefrom(3); close(0); char *envp[] = {NULL}; - execve(filename, argv, envp); + execve(filename, argv.data(), envp); MERROR("Failed to execve: " << strerror(errno)); return -1; } diff --git a/src/common/stack_trace.cpp b/src/common/stack_trace.cpp index 141621427..8d4f8c6f1 100644 --- a/src/common/stack_trace.cpp +++ b/src/common/stack_trace.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/stack_trace.h b/src/common/stack_trace.h index 272fb89ae..ae6573885 100644 --- a/src/common/stack_trace.h +++ b/src/common/stack_trace.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index cbf7163c5..2748c798c 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/threadpool.h b/src/common/threadpool.h index a43e38a76..5e490ee7d 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/timings.cc b/src/common/timings.cc new file mode 100644 index 000000000..612ac2cc6 --- /dev/null +++ b/src/common/timings.cc @@ -0,0 +1,125 @@ +#include <string.h> +#include <errno.h> +#include <time.h> +#include <algorithm> +#include <boost/algorithm/string.hpp> +#include "misc_log_ex.h" +#include "timings.h" + +#define N_EXPECTED_FIELDS (8+11) + +TimingsDatabase::TimingsDatabase() +{ +} + +TimingsDatabase::TimingsDatabase(const std::string &filename): + filename(filename) +{ + load(); +} + +TimingsDatabase::~TimingsDatabase() +{ + save(); +} + +bool TimingsDatabase::load() +{ + instances.clear(); + + if (filename.empty()) + return true; + + FILE *f = fopen(filename.c_str(), "r"); + if (!f) + { + MDEBUG("Failed to load timings file " << filename << ": " << strerror(errno)); + return false; + } + while (1) + { + char s[4096]; + if (!fgets(s, sizeof(s), f)) + break; + char *tab = strchr(s, '\t'); + if (!tab) + { + MWARNING("Bad format: no tab found"); + continue; + } + const std::string name = std::string(s, tab - s); + std::vector<std::string> fields; + char *ptr = tab + 1; + boost::split(fields, ptr, boost::is_any_of(" ")); + if (fields.size() != N_EXPECTED_FIELDS) + { + MERROR("Bad format: wrong number of fields: got " << fields.size() << " expected " << N_EXPECTED_FIELDS); + continue; + } + + instance i; + + unsigned int idx = 0; + i.t = atoi(fields[idx++].c_str()); + i.npoints = atoi(fields[idx++].c_str()); + i.min = atof(fields[idx++].c_str()); + i.max = atof(fields[idx++].c_str()); + i.mean = atof(fields[idx++].c_str()); + i.median = atof(fields[idx++].c_str()); + i.stddev = atof(fields[idx++].c_str()); + i.npskew = atof(fields[idx++].c_str()); + i.deciles.reserve(11); + for (int n = 0; n < 11; ++n) + { + i.deciles.push_back(atoi(fields[idx++].c_str())); + } + instances.insert(std::make_pair(name, i)); + } + fclose(f); + return true; +} + +bool TimingsDatabase::save() +{ + if (filename.empty()) + return true; + + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to write to file " << filename << ": " << strerror(errno)); + return false; + } + for (const auto &i: instances) + { + fprintf(f, "%s", i.first.c_str()); + fprintf(f, "\t%lu", (unsigned long)i.second.t); + fprintf(f, " %zu", i.second.npoints); + fprintf(f, " %f", i.second.min); + fprintf(f, " %f", i.second.max); + fprintf(f, " %f", i.second.mean); + fprintf(f, " %f", i.second.median); + fprintf(f, " %f", i.second.stddev); + fprintf(f, " %f", i.second.npskew); + for (uint64_t v: i.second.deciles) + fprintf(f, " %lu", (unsigned long)v); + fputc('\n', f); + } + fclose(f); + return true; +} + +std::vector<TimingsDatabase::instance> TimingsDatabase::get(const char *name) const +{ + std::vector<instance> ret; + auto range = instances.equal_range(name); + for (auto i = range.first; i != range.second; ++i) + ret.push_back(i->second); + std::sort(ret.begin(), ret.end(), [](const instance &e0, const instance &e1){ return e0.t < e1.t; }); + return ret; +} + +void TimingsDatabase::add(const char *name, const instance &i) +{ + instances.insert(std::make_pair(name, i)); +} diff --git a/src/common/timings.h b/src/common/timings.h new file mode 100644 index 000000000..fb905611f --- /dev/null +++ b/src/common/timings.h @@ -0,0 +1,34 @@ +#pragma once + +#include <stdint.h> +#include <string> +#include <vector> +#include <map> + +class TimingsDatabase +{ +public: + struct instance + { + time_t t; + size_t npoints; + double min, max, mean, median, stddev, npskew; + std::vector<uint64_t> deciles; + }; + +public: + TimingsDatabase(); + TimingsDatabase(const std::string &filename); + ~TimingsDatabase(); + + std::vector<instance> get(const char *name) const; + void add(const char *name, const instance &data); + +private: + bool load(); + bool save(); + +private: + std::string filename; + std::multimap<std::string, instance> instances; +}; diff --git a/src/common/unordered_containers_boost_serialization.h b/src/common/unordered_containers_boost_serialization.h index d78dc6a30..74e2c3f81 100644 --- a/src/common/unordered_containers_boost_serialization.h +++ b/src/common/unordered_containers_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 9f12f8dbc..0bc6ff63c 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/updates.h b/src/common/updates.h index 6ec22f183..8fda6d207 100644 --- a/src/common/updates.h +++ b/src/common/updates.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/util.cpp b/src/common/util.cpp index 28745eea4..80b8a9e81 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/util.h b/src/common/util.h index d5aca15d1..ef2305bf4 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/common/varint.h b/src/common/varint.h index 151d13dbf..a0d79be28 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -123,6 +123,6 @@ namespace tools { */ template<typename InputIt, typename T> int read_varint(InputIt &&first, InputIt &&last, T &i) { - return read_varint<std::numeric_limits<T>::digits, InputIt, T>(std::move(first), std::move(last), i); + return read_varint<std::numeric_limits<T>::digits>(std::forward<InputIt>(first), std::forward<InputIt>(last), i); } } diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 0c635e7cb..84f002929 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -45,8 +45,13 @@ set(crypto_sources random.c skein.c slow-hash.c + CryptonightR_JIT.c tree-hash.c) +if(ARCH_ID STREQUAL "i386" OR ARCH_ID STREQUAL "x86_64" OR ARCH_ID STREQUAL "x86-64") +list(APPEND crypto_sources CryptonightR_template.S) +endif() + set(crypto_headers) set(crypto_private_headers @@ -66,7 +71,9 @@ set(crypto_private_headers oaes_lib.h random.h skein.h - skein_port.h) + skein_port.h + CryptonightR_JIT.h + CryptonightR_template.h) monero_private_headers(cncrypto ${crypto_private_headers}) @@ -101,4 +108,5 @@ if (ANDROID OR IOS) endif() endif() - +# cheat because cmake and ccache hate each other +set_property(SOURCE CryptonightR_template.S PROPERTY LANGUAGE C) diff --git a/src/crypto/CryptonightR_JIT.c b/src/crypto/CryptonightR_JIT.c new file mode 100644 index 000000000..68258a959 --- /dev/null +++ b/src/crypto/CryptonightR_JIT.c @@ -0,0 +1,110 @@ +#include <assert.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> +#include <stdio.h> +#include <unistd.h> + +#include "int-util.h" +#include "hash-ops.h" +#include "variant4_random_math.h" +#include "CryptonightR_JIT.h" +#include "CryptonightR_template.h" + +static const uint8_t prologue[] = { +#if defined __i386 || defined __x86_64__ + 0x4C, 0x8B, 0xD7, // mov r10, rdi + 0x53, // push rbx + 0x55, // push rbp + 0x41, 0x57, // push r15 + 0x4C, 0x8B, 0xDC, // mov r11, rsp + 0x41, 0x8B, 0x1A, // mov ebx, DWORD PTR [r10] + 0x41, 0x8B, 0x72, 0x04, // mov esi, DWORD PTR [r10+4] + 0x41, 0x8B, 0x7A, 0x08, // mov edi, DWORD PTR [r10+8] + 0x41, 0x8B, 0x6A, 0x0C, // mov ebp, DWORD PTR [r10+12] + 0x41, 0x8B, 0x62, 0x10, // mov esp, DWORD PTR [r10+16] + 0x45, 0x8B, 0x7A, 0x14, // mov r15d, DWORD PTR [r10+20] + 0x41, 0x8B, 0x42, 0x18, // mov eax, DWORD PTR [r10+24] + 0x41, 0x8B, 0x52, 0x1C, // mov edx, DWORD PTR [r10+28] + 0x45, 0x8B, 0x4A, 0x20, // mov r9d, DWORD PTR [r10+32] +#endif +}; + +static const uint8_t epilogue[] = { +#if defined __i386 || defined __x86_64__ + 0x49, 0x8B, 0xE3, // mov rsp, r11 + 0x41, 0x89, 0x1A, // mov DWORD PTR [r10], ebx + 0x41, 0x89, 0x72, 0x04, // mov DWORD PTR [r10+4], esi + 0x41, 0x89, 0x7A, 0x08, // mov DWORD PTR [r10+8], edi + 0x41, 0x89, 0x6A, 0x0C, // mov DWORD PTR [r10+12], ebp + 0x41, 0x5F, // pop r15 + 0x5D, // pop rbp + 0x5B, // pop rbx + 0xC3, // ret +#endif +}; + +#define APPEND_CODE(src, size) \ + do { \ + if (JIT_code + (size) > JIT_code_end) \ + return -1; \ + memcpy(JIT_code, (src), (size)); \ + JIT_code += (size); \ + } while (0) + +int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_func buf, const size_t buf_size) +{ +#if defined __i386 || defined __x86_64__ + uint8_t* JIT_code = (uint8_t*) buf; + const uint8_t* JIT_code_end = JIT_code + buf_size; + + APPEND_CODE(prologue, sizeof(prologue)); + + uint32_t prev_rot_src = 0xFFFFFFFFU; + + for (int i = 0;; ++i) + { + const struct V4_Instruction inst = code[i]; + if (inst.opcode == RET) + break; + + const uint8_t opcode = (inst.opcode == MUL) ? inst.opcode : (inst.opcode + 2); + + const uint32_t a = inst.dst_index; + const uint32_t b = inst.src_index; + const uint8_t c = opcode | (inst.dst_index << V4_OPCODE_BITS) | (((inst.src_index == 8) ? inst.dst_index : inst.src_index) << (V4_OPCODE_BITS + V4_DST_INDEX_BITS)); + + switch (inst.opcode) + { + case ROR: + case ROL: + if (b != prev_rot_src) + { + prev_rot_src = b; + const uint8_t* p1 = (const uint8_t*) instructions_mov[c]; + const uint8_t* p2 = (const uint8_t*) instructions_mov[c + 1]; + APPEND_CODE(p1, p2 - p1); + } + break; + } + + if (a == prev_rot_src) + prev_rot_src = 0xFFFFFFFFU; + + const uint8_t* p1 = (const uint8_t*) instructions[c]; + const uint8_t* p2 = (const uint8_t*) instructions[c + 1]; + APPEND_CODE(p1, p2 - p1); + + if (inst.opcode == ADD) + *(uint32_t*)(JIT_code - 4) = inst.C; + } + + APPEND_CODE(epilogue, sizeof(epilogue)); + + __builtin___clear_cache((char*)buf, (char*)JIT_code); + + return 0; +#else + return 1; +#endif +} diff --git a/src/crypto/CryptonightR_JIT.h b/src/crypto/CryptonightR_JIT.h new file mode 100644 index 000000000..cb32c3a79 --- /dev/null +++ b/src/crypto/CryptonightR_JIT.h @@ -0,0 +1,22 @@ +#ifndef CRYPTONIGHTR_JIT_H +#define CRYPTONIGHTR_JIT_H + +// Minimalistic JIT code generator for random math sequence in CryptonightR +// +// Usage: +// - Allocate writable and executable memory +// - Call v4_generate_JIT_code with "buf" pointed to memory allocated on previous step +// - Call the generated code instead of "v4_random_math(code, r)", omit the "code" parameter + +typedef void (*v4_random_math_JIT_func)(uint32_t* r) +#if defined __i386 || defined __x86_64__ +__attribute__((sysv_abi)) +#endif +; + +// Given the random math sequence, generates machine code (x86-64) for it +// Returns 0 if code was generated successfully +// Returns -1 if provided buffer was too small +int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_func buf, const size_t buf_size); + +#endif // CRYPTONIGHTR_JIT_H diff --git a/src/crypto/CryptonightR_template.S b/src/crypto/CryptonightR_template.S new file mode 100644 index 000000000..068de22ec --- /dev/null +++ b/src/crypto/CryptonightR_template.S @@ -0,0 +1,1590 @@ +#ifdef __APPLE__ +# define ALIGN(x) .align 6 +#else +# define ALIGN(x) .align 64 +#endif +.intel_syntax noprefix +#ifdef __APPLE__ +# define FN_PREFIX(fn) _ ## fn +.text +#else +# define FN_PREFIX(fn) fn +.section .text +#endif + +#define PUBLIC .global + +PUBLIC FN_PREFIX(CryptonightR_instruction0) +PUBLIC FN_PREFIX(CryptonightR_instruction1) +PUBLIC FN_PREFIX(CryptonightR_instruction2) +PUBLIC FN_PREFIX(CryptonightR_instruction3) +PUBLIC FN_PREFIX(CryptonightR_instruction4) +PUBLIC FN_PREFIX(CryptonightR_instruction5) +PUBLIC FN_PREFIX(CryptonightR_instruction6) +PUBLIC FN_PREFIX(CryptonightR_instruction7) +PUBLIC FN_PREFIX(CryptonightR_instruction8) +PUBLIC FN_PREFIX(CryptonightR_instruction9) +PUBLIC FN_PREFIX(CryptonightR_instruction10) +PUBLIC FN_PREFIX(CryptonightR_instruction11) +PUBLIC FN_PREFIX(CryptonightR_instruction12) +PUBLIC FN_PREFIX(CryptonightR_instruction13) +PUBLIC FN_PREFIX(CryptonightR_instruction14) +PUBLIC FN_PREFIX(CryptonightR_instruction15) +PUBLIC FN_PREFIX(CryptonightR_instruction16) +PUBLIC FN_PREFIX(CryptonightR_instruction17) +PUBLIC FN_PREFIX(CryptonightR_instruction18) +PUBLIC FN_PREFIX(CryptonightR_instruction19) +PUBLIC FN_PREFIX(CryptonightR_instruction20) +PUBLIC FN_PREFIX(CryptonightR_instruction21) +PUBLIC FN_PREFIX(CryptonightR_instruction22) +PUBLIC FN_PREFIX(CryptonightR_instruction23) +PUBLIC FN_PREFIX(CryptonightR_instruction24) +PUBLIC FN_PREFIX(CryptonightR_instruction25) +PUBLIC FN_PREFIX(CryptonightR_instruction26) +PUBLIC FN_PREFIX(CryptonightR_instruction27) +PUBLIC FN_PREFIX(CryptonightR_instruction28) +PUBLIC FN_PREFIX(CryptonightR_instruction29) +PUBLIC FN_PREFIX(CryptonightR_instruction30) +PUBLIC FN_PREFIX(CryptonightR_instruction31) +PUBLIC FN_PREFIX(CryptonightR_instruction32) +PUBLIC FN_PREFIX(CryptonightR_instruction33) +PUBLIC FN_PREFIX(CryptonightR_instruction34) +PUBLIC FN_PREFIX(CryptonightR_instruction35) +PUBLIC FN_PREFIX(CryptonightR_instruction36) +PUBLIC FN_PREFIX(CryptonightR_instruction37) +PUBLIC FN_PREFIX(CryptonightR_instruction38) +PUBLIC FN_PREFIX(CryptonightR_instruction39) +PUBLIC FN_PREFIX(CryptonightR_instruction40) +PUBLIC FN_PREFIX(CryptonightR_instruction41) +PUBLIC FN_PREFIX(CryptonightR_instruction42) +PUBLIC FN_PREFIX(CryptonightR_instruction43) +PUBLIC FN_PREFIX(CryptonightR_instruction44) +PUBLIC FN_PREFIX(CryptonightR_instruction45) +PUBLIC FN_PREFIX(CryptonightR_instruction46) +PUBLIC FN_PREFIX(CryptonightR_instruction47) +PUBLIC FN_PREFIX(CryptonightR_instruction48) +PUBLIC FN_PREFIX(CryptonightR_instruction49) +PUBLIC FN_PREFIX(CryptonightR_instruction50) +PUBLIC FN_PREFIX(CryptonightR_instruction51) +PUBLIC FN_PREFIX(CryptonightR_instruction52) +PUBLIC FN_PREFIX(CryptonightR_instruction53) +PUBLIC FN_PREFIX(CryptonightR_instruction54) +PUBLIC FN_PREFIX(CryptonightR_instruction55) +PUBLIC FN_PREFIX(CryptonightR_instruction56) +PUBLIC FN_PREFIX(CryptonightR_instruction57) +PUBLIC FN_PREFIX(CryptonightR_instruction58) +PUBLIC FN_PREFIX(CryptonightR_instruction59) +PUBLIC FN_PREFIX(CryptonightR_instruction60) +PUBLIC FN_PREFIX(CryptonightR_instruction61) +PUBLIC FN_PREFIX(CryptonightR_instruction62) +PUBLIC FN_PREFIX(CryptonightR_instruction63) +PUBLIC FN_PREFIX(CryptonightR_instruction64) +PUBLIC FN_PREFIX(CryptonightR_instruction65) +PUBLIC FN_PREFIX(CryptonightR_instruction66) +PUBLIC FN_PREFIX(CryptonightR_instruction67) +PUBLIC FN_PREFIX(CryptonightR_instruction68) +PUBLIC FN_PREFIX(CryptonightR_instruction69) +PUBLIC FN_PREFIX(CryptonightR_instruction70) +PUBLIC FN_PREFIX(CryptonightR_instruction71) +PUBLIC FN_PREFIX(CryptonightR_instruction72) +PUBLIC FN_PREFIX(CryptonightR_instruction73) +PUBLIC FN_PREFIX(CryptonightR_instruction74) +PUBLIC FN_PREFIX(CryptonightR_instruction75) +PUBLIC FN_PREFIX(CryptonightR_instruction76) +PUBLIC FN_PREFIX(CryptonightR_instruction77) +PUBLIC FN_PREFIX(CryptonightR_instruction78) +PUBLIC FN_PREFIX(CryptonightR_instruction79) +PUBLIC FN_PREFIX(CryptonightR_instruction80) +PUBLIC FN_PREFIX(CryptonightR_instruction81) +PUBLIC FN_PREFIX(CryptonightR_instruction82) +PUBLIC FN_PREFIX(CryptonightR_instruction83) +PUBLIC FN_PREFIX(CryptonightR_instruction84) +PUBLIC FN_PREFIX(CryptonightR_instruction85) +PUBLIC FN_PREFIX(CryptonightR_instruction86) +PUBLIC FN_PREFIX(CryptonightR_instruction87) +PUBLIC FN_PREFIX(CryptonightR_instruction88) +PUBLIC FN_PREFIX(CryptonightR_instruction89) +PUBLIC FN_PREFIX(CryptonightR_instruction90) +PUBLIC FN_PREFIX(CryptonightR_instruction91) +PUBLIC FN_PREFIX(CryptonightR_instruction92) +PUBLIC FN_PREFIX(CryptonightR_instruction93) +PUBLIC FN_PREFIX(CryptonightR_instruction94) +PUBLIC FN_PREFIX(CryptonightR_instruction95) +PUBLIC FN_PREFIX(CryptonightR_instruction96) +PUBLIC FN_PREFIX(CryptonightR_instruction97) +PUBLIC FN_PREFIX(CryptonightR_instruction98) +PUBLIC FN_PREFIX(CryptonightR_instruction99) +PUBLIC FN_PREFIX(CryptonightR_instruction100) +PUBLIC FN_PREFIX(CryptonightR_instruction101) +PUBLIC FN_PREFIX(CryptonightR_instruction102) +PUBLIC FN_PREFIX(CryptonightR_instruction103) +PUBLIC FN_PREFIX(CryptonightR_instruction104) +PUBLIC FN_PREFIX(CryptonightR_instruction105) +PUBLIC FN_PREFIX(CryptonightR_instruction106) +PUBLIC FN_PREFIX(CryptonightR_instruction107) +PUBLIC FN_PREFIX(CryptonightR_instruction108) +PUBLIC FN_PREFIX(CryptonightR_instruction109) +PUBLIC FN_PREFIX(CryptonightR_instruction110) +PUBLIC FN_PREFIX(CryptonightR_instruction111) +PUBLIC FN_PREFIX(CryptonightR_instruction112) +PUBLIC FN_PREFIX(CryptonightR_instruction113) +PUBLIC FN_PREFIX(CryptonightR_instruction114) +PUBLIC FN_PREFIX(CryptonightR_instruction115) +PUBLIC FN_PREFIX(CryptonightR_instruction116) +PUBLIC FN_PREFIX(CryptonightR_instruction117) +PUBLIC FN_PREFIX(CryptonightR_instruction118) +PUBLIC FN_PREFIX(CryptonightR_instruction119) +PUBLIC FN_PREFIX(CryptonightR_instruction120) +PUBLIC FN_PREFIX(CryptonightR_instruction121) +PUBLIC FN_PREFIX(CryptonightR_instruction122) +PUBLIC FN_PREFIX(CryptonightR_instruction123) +PUBLIC FN_PREFIX(CryptonightR_instruction124) +PUBLIC FN_PREFIX(CryptonightR_instruction125) +PUBLIC FN_PREFIX(CryptonightR_instruction126) +PUBLIC FN_PREFIX(CryptonightR_instruction127) +PUBLIC FN_PREFIX(CryptonightR_instruction128) +PUBLIC FN_PREFIX(CryptonightR_instruction129) +PUBLIC FN_PREFIX(CryptonightR_instruction130) +PUBLIC FN_PREFIX(CryptonightR_instruction131) +PUBLIC FN_PREFIX(CryptonightR_instruction132) +PUBLIC FN_PREFIX(CryptonightR_instruction133) +PUBLIC FN_PREFIX(CryptonightR_instruction134) +PUBLIC FN_PREFIX(CryptonightR_instruction135) +PUBLIC FN_PREFIX(CryptonightR_instruction136) +PUBLIC FN_PREFIX(CryptonightR_instruction137) +PUBLIC FN_PREFIX(CryptonightR_instruction138) +PUBLIC FN_PREFIX(CryptonightR_instruction139) +PUBLIC FN_PREFIX(CryptonightR_instruction140) +PUBLIC FN_PREFIX(CryptonightR_instruction141) +PUBLIC FN_PREFIX(CryptonightR_instruction142) +PUBLIC FN_PREFIX(CryptonightR_instruction143) +PUBLIC FN_PREFIX(CryptonightR_instruction144) +PUBLIC FN_PREFIX(CryptonightR_instruction145) +PUBLIC FN_PREFIX(CryptonightR_instruction146) +PUBLIC FN_PREFIX(CryptonightR_instruction147) +PUBLIC FN_PREFIX(CryptonightR_instruction148) +PUBLIC FN_PREFIX(CryptonightR_instruction149) +PUBLIC FN_PREFIX(CryptonightR_instruction150) +PUBLIC FN_PREFIX(CryptonightR_instruction151) +PUBLIC FN_PREFIX(CryptonightR_instruction152) +PUBLIC FN_PREFIX(CryptonightR_instruction153) +PUBLIC FN_PREFIX(CryptonightR_instruction154) +PUBLIC FN_PREFIX(CryptonightR_instruction155) +PUBLIC FN_PREFIX(CryptonightR_instruction156) +PUBLIC FN_PREFIX(CryptonightR_instruction157) +PUBLIC FN_PREFIX(CryptonightR_instruction158) +PUBLIC FN_PREFIX(CryptonightR_instruction159) +PUBLIC FN_PREFIX(CryptonightR_instruction160) +PUBLIC FN_PREFIX(CryptonightR_instruction161) +PUBLIC FN_PREFIX(CryptonightR_instruction162) +PUBLIC FN_PREFIX(CryptonightR_instruction163) +PUBLIC FN_PREFIX(CryptonightR_instruction164) +PUBLIC FN_PREFIX(CryptonightR_instruction165) +PUBLIC FN_PREFIX(CryptonightR_instruction166) +PUBLIC FN_PREFIX(CryptonightR_instruction167) +PUBLIC FN_PREFIX(CryptonightR_instruction168) +PUBLIC FN_PREFIX(CryptonightR_instruction169) +PUBLIC FN_PREFIX(CryptonightR_instruction170) +PUBLIC FN_PREFIX(CryptonightR_instruction171) +PUBLIC FN_PREFIX(CryptonightR_instruction172) +PUBLIC FN_PREFIX(CryptonightR_instruction173) +PUBLIC FN_PREFIX(CryptonightR_instruction174) +PUBLIC FN_PREFIX(CryptonightR_instruction175) +PUBLIC FN_PREFIX(CryptonightR_instruction176) +PUBLIC FN_PREFIX(CryptonightR_instruction177) +PUBLIC FN_PREFIX(CryptonightR_instruction178) +PUBLIC FN_PREFIX(CryptonightR_instruction179) +PUBLIC FN_PREFIX(CryptonightR_instruction180) +PUBLIC FN_PREFIX(CryptonightR_instruction181) +PUBLIC FN_PREFIX(CryptonightR_instruction182) +PUBLIC FN_PREFIX(CryptonightR_instruction183) +PUBLIC FN_PREFIX(CryptonightR_instruction184) +PUBLIC FN_PREFIX(CryptonightR_instruction185) +PUBLIC FN_PREFIX(CryptonightR_instruction186) +PUBLIC FN_PREFIX(CryptonightR_instruction187) +PUBLIC FN_PREFIX(CryptonightR_instruction188) +PUBLIC FN_PREFIX(CryptonightR_instruction189) +PUBLIC FN_PREFIX(CryptonightR_instruction190) +PUBLIC FN_PREFIX(CryptonightR_instruction191) +PUBLIC FN_PREFIX(CryptonightR_instruction192) +PUBLIC FN_PREFIX(CryptonightR_instruction193) +PUBLIC FN_PREFIX(CryptonightR_instruction194) +PUBLIC FN_PREFIX(CryptonightR_instruction195) +PUBLIC FN_PREFIX(CryptonightR_instruction196) +PUBLIC FN_PREFIX(CryptonightR_instruction197) +PUBLIC FN_PREFIX(CryptonightR_instruction198) +PUBLIC FN_PREFIX(CryptonightR_instruction199) +PUBLIC FN_PREFIX(CryptonightR_instruction200) +PUBLIC FN_PREFIX(CryptonightR_instruction201) +PUBLIC FN_PREFIX(CryptonightR_instruction202) +PUBLIC FN_PREFIX(CryptonightR_instruction203) +PUBLIC FN_PREFIX(CryptonightR_instruction204) +PUBLIC FN_PREFIX(CryptonightR_instruction205) +PUBLIC FN_PREFIX(CryptonightR_instruction206) +PUBLIC FN_PREFIX(CryptonightR_instruction207) +PUBLIC FN_PREFIX(CryptonightR_instruction208) +PUBLIC FN_PREFIX(CryptonightR_instruction209) +PUBLIC FN_PREFIX(CryptonightR_instruction210) +PUBLIC FN_PREFIX(CryptonightR_instruction211) +PUBLIC FN_PREFIX(CryptonightR_instruction212) +PUBLIC FN_PREFIX(CryptonightR_instruction213) +PUBLIC FN_PREFIX(CryptonightR_instruction214) +PUBLIC FN_PREFIX(CryptonightR_instruction215) +PUBLIC FN_PREFIX(CryptonightR_instruction216) +PUBLIC FN_PREFIX(CryptonightR_instruction217) +PUBLIC FN_PREFIX(CryptonightR_instruction218) +PUBLIC FN_PREFIX(CryptonightR_instruction219) +PUBLIC FN_PREFIX(CryptonightR_instruction220) +PUBLIC FN_PREFIX(CryptonightR_instruction221) +PUBLIC FN_PREFIX(CryptonightR_instruction222) +PUBLIC FN_PREFIX(CryptonightR_instruction223) +PUBLIC FN_PREFIX(CryptonightR_instruction224) +PUBLIC FN_PREFIX(CryptonightR_instruction225) +PUBLIC FN_PREFIX(CryptonightR_instruction226) +PUBLIC FN_PREFIX(CryptonightR_instruction227) +PUBLIC FN_PREFIX(CryptonightR_instruction228) +PUBLIC FN_PREFIX(CryptonightR_instruction229) +PUBLIC FN_PREFIX(CryptonightR_instruction230) +PUBLIC FN_PREFIX(CryptonightR_instruction231) +PUBLIC FN_PREFIX(CryptonightR_instruction232) +PUBLIC FN_PREFIX(CryptonightR_instruction233) +PUBLIC FN_PREFIX(CryptonightR_instruction234) +PUBLIC FN_PREFIX(CryptonightR_instruction235) +PUBLIC FN_PREFIX(CryptonightR_instruction236) +PUBLIC FN_PREFIX(CryptonightR_instruction237) +PUBLIC FN_PREFIX(CryptonightR_instruction238) +PUBLIC FN_PREFIX(CryptonightR_instruction239) +PUBLIC FN_PREFIX(CryptonightR_instruction240) +PUBLIC FN_PREFIX(CryptonightR_instruction241) +PUBLIC FN_PREFIX(CryptonightR_instruction242) +PUBLIC FN_PREFIX(CryptonightR_instruction243) +PUBLIC FN_PREFIX(CryptonightR_instruction244) +PUBLIC FN_PREFIX(CryptonightR_instruction245) +PUBLIC FN_PREFIX(CryptonightR_instruction246) +PUBLIC FN_PREFIX(CryptonightR_instruction247) +PUBLIC FN_PREFIX(CryptonightR_instruction248) +PUBLIC FN_PREFIX(CryptonightR_instruction249) +PUBLIC FN_PREFIX(CryptonightR_instruction250) +PUBLIC FN_PREFIX(CryptonightR_instruction251) +PUBLIC FN_PREFIX(CryptonightR_instruction252) +PUBLIC FN_PREFIX(CryptonightR_instruction253) +PUBLIC FN_PREFIX(CryptonightR_instruction254) +PUBLIC FN_PREFIX(CryptonightR_instruction255) +PUBLIC FN_PREFIX(CryptonightR_instruction256) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov0) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov1) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov2) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov3) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov4) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov5) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov6) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov7) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov8) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov9) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov10) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov11) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov12) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov13) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov14) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov15) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov16) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov17) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov18) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov19) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov20) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov21) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov22) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov23) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov24) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov25) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov26) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov27) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov28) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov29) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov30) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov31) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov32) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov33) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov34) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov35) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov36) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov37) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov38) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov39) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov40) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov41) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov42) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov43) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov44) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov45) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov46) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov47) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov48) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov49) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov50) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov51) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov52) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov53) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov54) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov55) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov56) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov57) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov58) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov59) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov60) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov61) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov62) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov63) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov64) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov65) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov66) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov67) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov68) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov69) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov70) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov71) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov72) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov73) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov74) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov75) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov76) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov77) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov78) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov79) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov80) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov81) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov82) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov83) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov84) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov85) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov86) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov87) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov88) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov89) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov90) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov91) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov92) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov93) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov94) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov95) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov96) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov97) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov98) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov99) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov100) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov101) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov102) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov103) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov104) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov105) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov106) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov107) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov108) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov109) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov110) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov111) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov112) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov113) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov114) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov115) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov116) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov117) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov118) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov119) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov120) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov121) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov122) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov123) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov124) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov125) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov126) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov127) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov128) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov129) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov130) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov131) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov132) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov133) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov134) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov135) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov136) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov137) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov138) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov139) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov140) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov141) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov142) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov143) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov144) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov145) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov146) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov147) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov148) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov149) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov150) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov151) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov152) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov153) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov154) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov155) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov156) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov157) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov158) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov159) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov160) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov161) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov162) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov163) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov164) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov165) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov166) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov167) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov168) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov169) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov170) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov171) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov172) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov173) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov174) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov175) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov176) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov177) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov178) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov179) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov180) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov181) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov182) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov183) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov184) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov185) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov186) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov187) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov188) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov189) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov190) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov191) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov192) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov193) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov194) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov195) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov196) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov197) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov198) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov199) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov200) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov201) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov202) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov203) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov204) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov205) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov206) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov207) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov208) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov209) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov210) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov211) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov212) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov213) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov214) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov215) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov216) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov217) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov218) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov219) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov220) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov221) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov222) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov223) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov224) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov225) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov226) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov227) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov228) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov229) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov230) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov231) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov232) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov233) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov234) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov235) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov236) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov237) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov238) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov239) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov240) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov241) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov242) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov243) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov244) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov245) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov246) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov247) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov248) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov249) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov250) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov251) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov252) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov253) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov254) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov255) +PUBLIC FN_PREFIX(CryptonightR_instruction_mov256) + +FN_PREFIX(CryptonightR_instruction0): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction1): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction2): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction3): + add ebx, r9d + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction4): + sub ebx, r9d +FN_PREFIX(CryptonightR_instruction5): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction6): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction7): + xor ebx, r9d +FN_PREFIX(CryptonightR_instruction8): + imul esi, ebx +FN_PREFIX(CryptonightR_instruction9): + imul esi, ebx +FN_PREFIX(CryptonightR_instruction10): + imul esi, ebx +FN_PREFIX(CryptonightR_instruction11): + add esi, ebx + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction12): + sub esi, ebx +FN_PREFIX(CryptonightR_instruction13): + ror esi, cl +FN_PREFIX(CryptonightR_instruction14): + rol esi, cl +FN_PREFIX(CryptonightR_instruction15): + xor esi, ebx +FN_PREFIX(CryptonightR_instruction16): + imul edi, ebx +FN_PREFIX(CryptonightR_instruction17): + imul edi, ebx +FN_PREFIX(CryptonightR_instruction18): + imul edi, ebx +FN_PREFIX(CryptonightR_instruction19): + add edi, ebx + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction20): + sub edi, ebx +FN_PREFIX(CryptonightR_instruction21): + ror edi, cl +FN_PREFIX(CryptonightR_instruction22): + rol edi, cl +FN_PREFIX(CryptonightR_instruction23): + xor edi, ebx +FN_PREFIX(CryptonightR_instruction24): + imul ebp, ebx +FN_PREFIX(CryptonightR_instruction25): + imul ebp, ebx +FN_PREFIX(CryptonightR_instruction26): + imul ebp, ebx +FN_PREFIX(CryptonightR_instruction27): + add ebp, ebx + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction28): + sub ebp, ebx +FN_PREFIX(CryptonightR_instruction29): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction30): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction31): + xor ebp, ebx +FN_PREFIX(CryptonightR_instruction32): + imul ebx, esi +FN_PREFIX(CryptonightR_instruction33): + imul ebx, esi +FN_PREFIX(CryptonightR_instruction34): + imul ebx, esi +FN_PREFIX(CryptonightR_instruction35): + add ebx, esi + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction36): + sub ebx, esi +FN_PREFIX(CryptonightR_instruction37): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction38): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction39): + xor ebx, esi +FN_PREFIX(CryptonightR_instruction40): + imul esi, esi +FN_PREFIX(CryptonightR_instruction41): + imul esi, esi +FN_PREFIX(CryptonightR_instruction42): + imul esi, esi +FN_PREFIX(CryptonightR_instruction43): + add esi, r9d + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction44): + sub esi, r9d +FN_PREFIX(CryptonightR_instruction45): + ror esi, cl +FN_PREFIX(CryptonightR_instruction46): + rol esi, cl +FN_PREFIX(CryptonightR_instruction47): + xor esi, r9d +FN_PREFIX(CryptonightR_instruction48): + imul edi, esi +FN_PREFIX(CryptonightR_instruction49): + imul edi, esi +FN_PREFIX(CryptonightR_instruction50): + imul edi, esi +FN_PREFIX(CryptonightR_instruction51): + add edi, esi + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction52): + sub edi, esi +FN_PREFIX(CryptonightR_instruction53): + ror edi, cl +FN_PREFIX(CryptonightR_instruction54): + rol edi, cl +FN_PREFIX(CryptonightR_instruction55): + xor edi, esi +FN_PREFIX(CryptonightR_instruction56): + imul ebp, esi +FN_PREFIX(CryptonightR_instruction57): + imul ebp, esi +FN_PREFIX(CryptonightR_instruction58): + imul ebp, esi +FN_PREFIX(CryptonightR_instruction59): + add ebp, esi + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction60): + sub ebp, esi +FN_PREFIX(CryptonightR_instruction61): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction62): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction63): + xor ebp, esi +FN_PREFIX(CryptonightR_instruction64): + imul ebx, edi +FN_PREFIX(CryptonightR_instruction65): + imul ebx, edi +FN_PREFIX(CryptonightR_instruction66): + imul ebx, edi +FN_PREFIX(CryptonightR_instruction67): + add ebx, edi + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction68): + sub ebx, edi +FN_PREFIX(CryptonightR_instruction69): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction70): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction71): + xor ebx, edi +FN_PREFIX(CryptonightR_instruction72): + imul esi, edi +FN_PREFIX(CryptonightR_instruction73): + imul esi, edi +FN_PREFIX(CryptonightR_instruction74): + imul esi, edi +FN_PREFIX(CryptonightR_instruction75): + add esi, edi + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction76): + sub esi, edi +FN_PREFIX(CryptonightR_instruction77): + ror esi, cl +FN_PREFIX(CryptonightR_instruction78): + rol esi, cl +FN_PREFIX(CryptonightR_instruction79): + xor esi, edi +FN_PREFIX(CryptonightR_instruction80): + imul edi, edi +FN_PREFIX(CryptonightR_instruction81): + imul edi, edi +FN_PREFIX(CryptonightR_instruction82): + imul edi, edi +FN_PREFIX(CryptonightR_instruction83): + add edi, r9d + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction84): + sub edi, r9d +FN_PREFIX(CryptonightR_instruction85): + ror edi, cl +FN_PREFIX(CryptonightR_instruction86): + rol edi, cl +FN_PREFIX(CryptonightR_instruction87): + xor edi, r9d +FN_PREFIX(CryptonightR_instruction88): + imul ebp, edi +FN_PREFIX(CryptonightR_instruction89): + imul ebp, edi +FN_PREFIX(CryptonightR_instruction90): + imul ebp, edi +FN_PREFIX(CryptonightR_instruction91): + add ebp, edi + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction92): + sub ebp, edi +FN_PREFIX(CryptonightR_instruction93): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction94): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction95): + xor ebp, edi +FN_PREFIX(CryptonightR_instruction96): + imul ebx, ebp +FN_PREFIX(CryptonightR_instruction97): + imul ebx, ebp +FN_PREFIX(CryptonightR_instruction98): + imul ebx, ebp +FN_PREFIX(CryptonightR_instruction99): + add ebx, ebp + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction100): + sub ebx, ebp +FN_PREFIX(CryptonightR_instruction101): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction102): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction103): + xor ebx, ebp +FN_PREFIX(CryptonightR_instruction104): + imul esi, ebp +FN_PREFIX(CryptonightR_instruction105): + imul esi, ebp +FN_PREFIX(CryptonightR_instruction106): + imul esi, ebp +FN_PREFIX(CryptonightR_instruction107): + add esi, ebp + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction108): + sub esi, ebp +FN_PREFIX(CryptonightR_instruction109): + ror esi, cl +FN_PREFIX(CryptonightR_instruction110): + rol esi, cl +FN_PREFIX(CryptonightR_instruction111): + xor esi, ebp +FN_PREFIX(CryptonightR_instruction112): + imul edi, ebp +FN_PREFIX(CryptonightR_instruction113): + imul edi, ebp +FN_PREFIX(CryptonightR_instruction114): + imul edi, ebp +FN_PREFIX(CryptonightR_instruction115): + add edi, ebp + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction116): + sub edi, ebp +FN_PREFIX(CryptonightR_instruction117): + ror edi, cl +FN_PREFIX(CryptonightR_instruction118): + rol edi, cl +FN_PREFIX(CryptonightR_instruction119): + xor edi, ebp +FN_PREFIX(CryptonightR_instruction120): + imul ebp, ebp +FN_PREFIX(CryptonightR_instruction121): + imul ebp, ebp +FN_PREFIX(CryptonightR_instruction122): + imul ebp, ebp +FN_PREFIX(CryptonightR_instruction123): + add ebp, r9d + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction124): + sub ebp, r9d +FN_PREFIX(CryptonightR_instruction125): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction126): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction127): + xor ebp, r9d +FN_PREFIX(CryptonightR_instruction128): + imul ebx, esp +FN_PREFIX(CryptonightR_instruction129): + imul ebx, esp +FN_PREFIX(CryptonightR_instruction130): + imul ebx, esp +FN_PREFIX(CryptonightR_instruction131): + add ebx, esp + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction132): + sub ebx, esp +FN_PREFIX(CryptonightR_instruction133): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction134): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction135): + xor ebx, esp +FN_PREFIX(CryptonightR_instruction136): + imul esi, esp +FN_PREFIX(CryptonightR_instruction137): + imul esi, esp +FN_PREFIX(CryptonightR_instruction138): + imul esi, esp +FN_PREFIX(CryptonightR_instruction139): + add esi, esp + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction140): + sub esi, esp +FN_PREFIX(CryptonightR_instruction141): + ror esi, cl +FN_PREFIX(CryptonightR_instruction142): + rol esi, cl +FN_PREFIX(CryptonightR_instruction143): + xor esi, esp +FN_PREFIX(CryptonightR_instruction144): + imul edi, esp +FN_PREFIX(CryptonightR_instruction145): + imul edi, esp +FN_PREFIX(CryptonightR_instruction146): + imul edi, esp +FN_PREFIX(CryptonightR_instruction147): + add edi, esp + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction148): + sub edi, esp +FN_PREFIX(CryptonightR_instruction149): + ror edi, cl +FN_PREFIX(CryptonightR_instruction150): + rol edi, cl +FN_PREFIX(CryptonightR_instruction151): + xor edi, esp +FN_PREFIX(CryptonightR_instruction152): + imul ebp, esp +FN_PREFIX(CryptonightR_instruction153): + imul ebp, esp +FN_PREFIX(CryptonightR_instruction154): + imul ebp, esp +FN_PREFIX(CryptonightR_instruction155): + add ebp, esp + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction156): + sub ebp, esp +FN_PREFIX(CryptonightR_instruction157): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction158): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction159): + xor ebp, esp +FN_PREFIX(CryptonightR_instruction160): + imul ebx, r15d +FN_PREFIX(CryptonightR_instruction161): + imul ebx, r15d +FN_PREFIX(CryptonightR_instruction162): + imul ebx, r15d +FN_PREFIX(CryptonightR_instruction163): + add ebx, r15d + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction164): + sub ebx, r15d +FN_PREFIX(CryptonightR_instruction165): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction166): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction167): + xor ebx, r15d +FN_PREFIX(CryptonightR_instruction168): + imul esi, r15d +FN_PREFIX(CryptonightR_instruction169): + imul esi, r15d +FN_PREFIX(CryptonightR_instruction170): + imul esi, r15d +FN_PREFIX(CryptonightR_instruction171): + add esi, r15d + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction172): + sub esi, r15d +FN_PREFIX(CryptonightR_instruction173): + ror esi, cl +FN_PREFIX(CryptonightR_instruction174): + rol esi, cl +FN_PREFIX(CryptonightR_instruction175): + xor esi, r15d +FN_PREFIX(CryptonightR_instruction176): + imul edi, r15d +FN_PREFIX(CryptonightR_instruction177): + imul edi, r15d +FN_PREFIX(CryptonightR_instruction178): + imul edi, r15d +FN_PREFIX(CryptonightR_instruction179): + add edi, r15d + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction180): + sub edi, r15d +FN_PREFIX(CryptonightR_instruction181): + ror edi, cl +FN_PREFIX(CryptonightR_instruction182): + rol edi, cl +FN_PREFIX(CryptonightR_instruction183): + xor edi, r15d +FN_PREFIX(CryptonightR_instruction184): + imul ebp, r15d +FN_PREFIX(CryptonightR_instruction185): + imul ebp, r15d +FN_PREFIX(CryptonightR_instruction186): + imul ebp, r15d +FN_PREFIX(CryptonightR_instruction187): + add ebp, r15d + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction188): + sub ebp, r15d +FN_PREFIX(CryptonightR_instruction189): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction190): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction191): + xor ebp, r15d +FN_PREFIX(CryptonightR_instruction192): + imul ebx, eax +FN_PREFIX(CryptonightR_instruction193): + imul ebx, eax +FN_PREFIX(CryptonightR_instruction194): + imul ebx, eax +FN_PREFIX(CryptonightR_instruction195): + add ebx, eax + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction196): + sub ebx, eax +FN_PREFIX(CryptonightR_instruction197): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction198): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction199): + xor ebx, eax +FN_PREFIX(CryptonightR_instruction200): + imul esi, eax +FN_PREFIX(CryptonightR_instruction201): + imul esi, eax +FN_PREFIX(CryptonightR_instruction202): + imul esi, eax +FN_PREFIX(CryptonightR_instruction203): + add esi, eax + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction204): + sub esi, eax +FN_PREFIX(CryptonightR_instruction205): + ror esi, cl +FN_PREFIX(CryptonightR_instruction206): + rol esi, cl +FN_PREFIX(CryptonightR_instruction207): + xor esi, eax +FN_PREFIX(CryptonightR_instruction208): + imul edi, eax +FN_PREFIX(CryptonightR_instruction209): + imul edi, eax +FN_PREFIX(CryptonightR_instruction210): + imul edi, eax +FN_PREFIX(CryptonightR_instruction211): + add edi, eax + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction212): + sub edi, eax +FN_PREFIX(CryptonightR_instruction213): + ror edi, cl +FN_PREFIX(CryptonightR_instruction214): + rol edi, cl +FN_PREFIX(CryptonightR_instruction215): + xor edi, eax +FN_PREFIX(CryptonightR_instruction216): + imul ebp, eax +FN_PREFIX(CryptonightR_instruction217): + imul ebp, eax +FN_PREFIX(CryptonightR_instruction218): + imul ebp, eax +FN_PREFIX(CryptonightR_instruction219): + add ebp, eax + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction220): + sub ebp, eax +FN_PREFIX(CryptonightR_instruction221): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction222): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction223): + xor ebp, eax +FN_PREFIX(CryptonightR_instruction224): + imul ebx, edx +FN_PREFIX(CryptonightR_instruction225): + imul ebx, edx +FN_PREFIX(CryptonightR_instruction226): + imul ebx, edx +FN_PREFIX(CryptonightR_instruction227): + add ebx, edx + add ebx, 2147483647 +FN_PREFIX(CryptonightR_instruction228): + sub ebx, edx +FN_PREFIX(CryptonightR_instruction229): + ror ebx, cl +FN_PREFIX(CryptonightR_instruction230): + rol ebx, cl +FN_PREFIX(CryptonightR_instruction231): + xor ebx, edx +FN_PREFIX(CryptonightR_instruction232): + imul esi, edx +FN_PREFIX(CryptonightR_instruction233): + imul esi, edx +FN_PREFIX(CryptonightR_instruction234): + imul esi, edx +FN_PREFIX(CryptonightR_instruction235): + add esi, edx + add esi, 2147483647 +FN_PREFIX(CryptonightR_instruction236): + sub esi, edx +FN_PREFIX(CryptonightR_instruction237): + ror esi, cl +FN_PREFIX(CryptonightR_instruction238): + rol esi, cl +FN_PREFIX(CryptonightR_instruction239): + xor esi, edx +FN_PREFIX(CryptonightR_instruction240): + imul edi, edx +FN_PREFIX(CryptonightR_instruction241): + imul edi, edx +FN_PREFIX(CryptonightR_instruction242): + imul edi, edx +FN_PREFIX(CryptonightR_instruction243): + add edi, edx + add edi, 2147483647 +FN_PREFIX(CryptonightR_instruction244): + sub edi, edx +FN_PREFIX(CryptonightR_instruction245): + ror edi, cl +FN_PREFIX(CryptonightR_instruction246): + rol edi, cl +FN_PREFIX(CryptonightR_instruction247): + xor edi, edx +FN_PREFIX(CryptonightR_instruction248): + imul ebp, edx +FN_PREFIX(CryptonightR_instruction249): + imul ebp, edx +FN_PREFIX(CryptonightR_instruction250): + imul ebp, edx +FN_PREFIX(CryptonightR_instruction251): + add ebp, edx + add ebp, 2147483647 +FN_PREFIX(CryptonightR_instruction252): + sub ebp, edx +FN_PREFIX(CryptonightR_instruction253): + ror ebp, cl +FN_PREFIX(CryptonightR_instruction254): + rol ebp, cl +FN_PREFIX(CryptonightR_instruction255): + xor ebp, edx +FN_PREFIX(CryptonightR_instruction256): + imul ebx, ebx +FN_PREFIX(CryptonightR_instruction_mov0): + +FN_PREFIX(CryptonightR_instruction_mov1): + +FN_PREFIX(CryptonightR_instruction_mov2): + +FN_PREFIX(CryptonightR_instruction_mov3): + +FN_PREFIX(CryptonightR_instruction_mov4): + +FN_PREFIX(CryptonightR_instruction_mov5): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov6): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov7): + +FN_PREFIX(CryptonightR_instruction_mov8): + +FN_PREFIX(CryptonightR_instruction_mov9): + +FN_PREFIX(CryptonightR_instruction_mov10): + +FN_PREFIX(CryptonightR_instruction_mov11): + +FN_PREFIX(CryptonightR_instruction_mov12): + +FN_PREFIX(CryptonightR_instruction_mov13): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov14): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov15): + +FN_PREFIX(CryptonightR_instruction_mov16): + +FN_PREFIX(CryptonightR_instruction_mov17): + +FN_PREFIX(CryptonightR_instruction_mov18): + +FN_PREFIX(CryptonightR_instruction_mov19): + +FN_PREFIX(CryptonightR_instruction_mov20): + +FN_PREFIX(CryptonightR_instruction_mov21): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov22): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov23): + +FN_PREFIX(CryptonightR_instruction_mov24): + +FN_PREFIX(CryptonightR_instruction_mov25): + +FN_PREFIX(CryptonightR_instruction_mov26): + +FN_PREFIX(CryptonightR_instruction_mov27): + +FN_PREFIX(CryptonightR_instruction_mov28): + +FN_PREFIX(CryptonightR_instruction_mov29): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov30): + mov ecx, ebx +FN_PREFIX(CryptonightR_instruction_mov31): + +FN_PREFIX(CryptonightR_instruction_mov32): + +FN_PREFIX(CryptonightR_instruction_mov33): + +FN_PREFIX(CryptonightR_instruction_mov34): + +FN_PREFIX(CryptonightR_instruction_mov35): + +FN_PREFIX(CryptonightR_instruction_mov36): + +FN_PREFIX(CryptonightR_instruction_mov37): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov38): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov39): + +FN_PREFIX(CryptonightR_instruction_mov40): + +FN_PREFIX(CryptonightR_instruction_mov41): + +FN_PREFIX(CryptonightR_instruction_mov42): + +FN_PREFIX(CryptonightR_instruction_mov43): + +FN_PREFIX(CryptonightR_instruction_mov44): + +FN_PREFIX(CryptonightR_instruction_mov45): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov46): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov47): + +FN_PREFIX(CryptonightR_instruction_mov48): + +FN_PREFIX(CryptonightR_instruction_mov49): + +FN_PREFIX(CryptonightR_instruction_mov50): + +FN_PREFIX(CryptonightR_instruction_mov51): + +FN_PREFIX(CryptonightR_instruction_mov52): + +FN_PREFIX(CryptonightR_instruction_mov53): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov54): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov55): + +FN_PREFIX(CryptonightR_instruction_mov56): + +FN_PREFIX(CryptonightR_instruction_mov57): + +FN_PREFIX(CryptonightR_instruction_mov58): + +FN_PREFIX(CryptonightR_instruction_mov59): + +FN_PREFIX(CryptonightR_instruction_mov60): + +FN_PREFIX(CryptonightR_instruction_mov61): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov62): + mov ecx, esi +FN_PREFIX(CryptonightR_instruction_mov63): + +FN_PREFIX(CryptonightR_instruction_mov64): + +FN_PREFIX(CryptonightR_instruction_mov65): + +FN_PREFIX(CryptonightR_instruction_mov66): + +FN_PREFIX(CryptonightR_instruction_mov67): + +FN_PREFIX(CryptonightR_instruction_mov68): + +FN_PREFIX(CryptonightR_instruction_mov69): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov70): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov71): + +FN_PREFIX(CryptonightR_instruction_mov72): + +FN_PREFIX(CryptonightR_instruction_mov73): + +FN_PREFIX(CryptonightR_instruction_mov74): + +FN_PREFIX(CryptonightR_instruction_mov75): + +FN_PREFIX(CryptonightR_instruction_mov76): + +FN_PREFIX(CryptonightR_instruction_mov77): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov78): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov79): + +FN_PREFIX(CryptonightR_instruction_mov80): + +FN_PREFIX(CryptonightR_instruction_mov81): + +FN_PREFIX(CryptonightR_instruction_mov82): + +FN_PREFIX(CryptonightR_instruction_mov83): + +FN_PREFIX(CryptonightR_instruction_mov84): + +FN_PREFIX(CryptonightR_instruction_mov85): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov86): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov87): + +FN_PREFIX(CryptonightR_instruction_mov88): + +FN_PREFIX(CryptonightR_instruction_mov89): + +FN_PREFIX(CryptonightR_instruction_mov90): + +FN_PREFIX(CryptonightR_instruction_mov91): + +FN_PREFIX(CryptonightR_instruction_mov92): + +FN_PREFIX(CryptonightR_instruction_mov93): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov94): + mov ecx, edi +FN_PREFIX(CryptonightR_instruction_mov95): + +FN_PREFIX(CryptonightR_instruction_mov96): + +FN_PREFIX(CryptonightR_instruction_mov97): + +FN_PREFIX(CryptonightR_instruction_mov98): + +FN_PREFIX(CryptonightR_instruction_mov99): + +FN_PREFIX(CryptonightR_instruction_mov100): + +FN_PREFIX(CryptonightR_instruction_mov101): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov102): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov103): + +FN_PREFIX(CryptonightR_instruction_mov104): + +FN_PREFIX(CryptonightR_instruction_mov105): + +FN_PREFIX(CryptonightR_instruction_mov106): + +FN_PREFIX(CryptonightR_instruction_mov107): + +FN_PREFIX(CryptonightR_instruction_mov108): + +FN_PREFIX(CryptonightR_instruction_mov109): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov110): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov111): + +FN_PREFIX(CryptonightR_instruction_mov112): + +FN_PREFIX(CryptonightR_instruction_mov113): + +FN_PREFIX(CryptonightR_instruction_mov114): + +FN_PREFIX(CryptonightR_instruction_mov115): + +FN_PREFIX(CryptonightR_instruction_mov116): + +FN_PREFIX(CryptonightR_instruction_mov117): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov118): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov119): + +FN_PREFIX(CryptonightR_instruction_mov120): + +FN_PREFIX(CryptonightR_instruction_mov121): + +FN_PREFIX(CryptonightR_instruction_mov122): + +FN_PREFIX(CryptonightR_instruction_mov123): + +FN_PREFIX(CryptonightR_instruction_mov124): + +FN_PREFIX(CryptonightR_instruction_mov125): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov126): + mov ecx, ebp +FN_PREFIX(CryptonightR_instruction_mov127): + +FN_PREFIX(CryptonightR_instruction_mov128): + +FN_PREFIX(CryptonightR_instruction_mov129): + +FN_PREFIX(CryptonightR_instruction_mov130): + +FN_PREFIX(CryptonightR_instruction_mov131): + +FN_PREFIX(CryptonightR_instruction_mov132): + +FN_PREFIX(CryptonightR_instruction_mov133): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov134): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov135): + +FN_PREFIX(CryptonightR_instruction_mov136): + +FN_PREFIX(CryptonightR_instruction_mov137): + +FN_PREFIX(CryptonightR_instruction_mov138): + +FN_PREFIX(CryptonightR_instruction_mov139): + +FN_PREFIX(CryptonightR_instruction_mov140): + +FN_PREFIX(CryptonightR_instruction_mov141): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov142): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov143): + +FN_PREFIX(CryptonightR_instruction_mov144): + +FN_PREFIX(CryptonightR_instruction_mov145): + +FN_PREFIX(CryptonightR_instruction_mov146): + +FN_PREFIX(CryptonightR_instruction_mov147): + +FN_PREFIX(CryptonightR_instruction_mov148): + +FN_PREFIX(CryptonightR_instruction_mov149): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov150): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov151): + +FN_PREFIX(CryptonightR_instruction_mov152): + +FN_PREFIX(CryptonightR_instruction_mov153): + +FN_PREFIX(CryptonightR_instruction_mov154): + +FN_PREFIX(CryptonightR_instruction_mov155): + +FN_PREFIX(CryptonightR_instruction_mov156): + +FN_PREFIX(CryptonightR_instruction_mov157): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov158): + mov ecx, esp +FN_PREFIX(CryptonightR_instruction_mov159): + +FN_PREFIX(CryptonightR_instruction_mov160): + +FN_PREFIX(CryptonightR_instruction_mov161): + +FN_PREFIX(CryptonightR_instruction_mov162): + +FN_PREFIX(CryptonightR_instruction_mov163): + +FN_PREFIX(CryptonightR_instruction_mov164): + +FN_PREFIX(CryptonightR_instruction_mov165): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov166): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov167): + +FN_PREFIX(CryptonightR_instruction_mov168): + +FN_PREFIX(CryptonightR_instruction_mov169): + +FN_PREFIX(CryptonightR_instruction_mov170): + +FN_PREFIX(CryptonightR_instruction_mov171): + +FN_PREFIX(CryptonightR_instruction_mov172): + +FN_PREFIX(CryptonightR_instruction_mov173): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov174): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov175): + +FN_PREFIX(CryptonightR_instruction_mov176): + +FN_PREFIX(CryptonightR_instruction_mov177): + +FN_PREFIX(CryptonightR_instruction_mov178): + +FN_PREFIX(CryptonightR_instruction_mov179): + +FN_PREFIX(CryptonightR_instruction_mov180): + +FN_PREFIX(CryptonightR_instruction_mov181): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov182): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov183): + +FN_PREFIX(CryptonightR_instruction_mov184): + +FN_PREFIX(CryptonightR_instruction_mov185): + +FN_PREFIX(CryptonightR_instruction_mov186): + +FN_PREFIX(CryptonightR_instruction_mov187): + +FN_PREFIX(CryptonightR_instruction_mov188): + +FN_PREFIX(CryptonightR_instruction_mov189): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov190): + mov ecx, r15d +FN_PREFIX(CryptonightR_instruction_mov191): + +FN_PREFIX(CryptonightR_instruction_mov192): + +FN_PREFIX(CryptonightR_instruction_mov193): + +FN_PREFIX(CryptonightR_instruction_mov194): + +FN_PREFIX(CryptonightR_instruction_mov195): + +FN_PREFIX(CryptonightR_instruction_mov196): + +FN_PREFIX(CryptonightR_instruction_mov197): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov198): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov199): + +FN_PREFIX(CryptonightR_instruction_mov200): + +FN_PREFIX(CryptonightR_instruction_mov201): + +FN_PREFIX(CryptonightR_instruction_mov202): + +FN_PREFIX(CryptonightR_instruction_mov203): + +FN_PREFIX(CryptonightR_instruction_mov204): + +FN_PREFIX(CryptonightR_instruction_mov205): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov206): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov207): + +FN_PREFIX(CryptonightR_instruction_mov208): + +FN_PREFIX(CryptonightR_instruction_mov209): + +FN_PREFIX(CryptonightR_instruction_mov210): + +FN_PREFIX(CryptonightR_instruction_mov211): + +FN_PREFIX(CryptonightR_instruction_mov212): + +FN_PREFIX(CryptonightR_instruction_mov213): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov214): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov215): + +FN_PREFIX(CryptonightR_instruction_mov216): + +FN_PREFIX(CryptonightR_instruction_mov217): + +FN_PREFIX(CryptonightR_instruction_mov218): + +FN_PREFIX(CryptonightR_instruction_mov219): + +FN_PREFIX(CryptonightR_instruction_mov220): + +FN_PREFIX(CryptonightR_instruction_mov221): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov222): + mov ecx, eax +FN_PREFIX(CryptonightR_instruction_mov223): + +FN_PREFIX(CryptonightR_instruction_mov224): + +FN_PREFIX(CryptonightR_instruction_mov225): + +FN_PREFIX(CryptonightR_instruction_mov226): + +FN_PREFIX(CryptonightR_instruction_mov227): + +FN_PREFIX(CryptonightR_instruction_mov228): + +FN_PREFIX(CryptonightR_instruction_mov229): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov230): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov231): + +FN_PREFIX(CryptonightR_instruction_mov232): + +FN_PREFIX(CryptonightR_instruction_mov233): + +FN_PREFIX(CryptonightR_instruction_mov234): + +FN_PREFIX(CryptonightR_instruction_mov235): + +FN_PREFIX(CryptonightR_instruction_mov236): + +FN_PREFIX(CryptonightR_instruction_mov237): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov238): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov239): + +FN_PREFIX(CryptonightR_instruction_mov240): + +FN_PREFIX(CryptonightR_instruction_mov241): + +FN_PREFIX(CryptonightR_instruction_mov242): + +FN_PREFIX(CryptonightR_instruction_mov243): + +FN_PREFIX(CryptonightR_instruction_mov244): + +FN_PREFIX(CryptonightR_instruction_mov245): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov246): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov247): + +FN_PREFIX(CryptonightR_instruction_mov248): + +FN_PREFIX(CryptonightR_instruction_mov249): + +FN_PREFIX(CryptonightR_instruction_mov250): + +FN_PREFIX(CryptonightR_instruction_mov251): + +FN_PREFIX(CryptonightR_instruction_mov252): + +FN_PREFIX(CryptonightR_instruction_mov253): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov254): + mov ecx, edx +FN_PREFIX(CryptonightR_instruction_mov255): + +FN_PREFIX(CryptonightR_instruction_mov256): diff --git a/src/crypto/CryptonightR_template.h b/src/crypto/CryptonightR_template.h new file mode 100644 index 000000000..dab1bd750 --- /dev/null +++ b/src/crypto/CryptonightR_template.h @@ -0,0 +1,1043 @@ +#ifndef CRYPTONIGHTR_TEMPLATE_H +#define CRYPTONIGHTR_TEMPLATE_H + +#if defined __i386 || defined __x86_64__ + +void CryptonightR_instruction0(void); +void CryptonightR_instruction1(void); +void CryptonightR_instruction2(void); +void CryptonightR_instruction3(void); +void CryptonightR_instruction4(void); +void CryptonightR_instruction5(void); +void CryptonightR_instruction6(void); +void CryptonightR_instruction7(void); +void CryptonightR_instruction8(void); +void CryptonightR_instruction9(void); +void CryptonightR_instruction10(void); +void CryptonightR_instruction11(void); +void CryptonightR_instruction12(void); +void CryptonightR_instruction13(void); +void CryptonightR_instruction14(void); +void CryptonightR_instruction15(void); +void CryptonightR_instruction16(void); +void CryptonightR_instruction17(void); +void CryptonightR_instruction18(void); +void CryptonightR_instruction19(void); +void CryptonightR_instruction20(void); +void CryptonightR_instruction21(void); +void CryptonightR_instruction22(void); +void CryptonightR_instruction23(void); +void CryptonightR_instruction24(void); +void CryptonightR_instruction25(void); +void CryptonightR_instruction26(void); +void CryptonightR_instruction27(void); +void CryptonightR_instruction28(void); +void CryptonightR_instruction29(void); +void CryptonightR_instruction30(void); +void CryptonightR_instruction31(void); +void CryptonightR_instruction32(void); +void CryptonightR_instruction33(void); +void CryptonightR_instruction34(void); +void CryptonightR_instruction35(void); +void CryptonightR_instruction36(void); +void CryptonightR_instruction37(void); +void CryptonightR_instruction38(void); +void CryptonightR_instruction39(void); +void CryptonightR_instruction40(void); +void CryptonightR_instruction41(void); +void CryptonightR_instruction42(void); +void CryptonightR_instruction43(void); +void CryptonightR_instruction44(void); +void CryptonightR_instruction45(void); +void CryptonightR_instruction46(void); +void CryptonightR_instruction47(void); +void CryptonightR_instruction48(void); +void CryptonightR_instruction49(void); +void CryptonightR_instruction50(void); +void CryptonightR_instruction51(void); +void CryptonightR_instruction52(void); +void CryptonightR_instruction53(void); +void CryptonightR_instruction54(void); +void CryptonightR_instruction55(void); +void CryptonightR_instruction56(void); +void CryptonightR_instruction57(void); +void CryptonightR_instruction58(void); +void CryptonightR_instruction59(void); +void CryptonightR_instruction60(void); +void CryptonightR_instruction61(void); +void CryptonightR_instruction62(void); +void CryptonightR_instruction63(void); +void CryptonightR_instruction64(void); +void CryptonightR_instruction65(void); +void CryptonightR_instruction66(void); +void CryptonightR_instruction67(void); +void CryptonightR_instruction68(void); +void CryptonightR_instruction69(void); +void CryptonightR_instruction70(void); +void CryptonightR_instruction71(void); +void CryptonightR_instruction72(void); +void CryptonightR_instruction73(void); +void CryptonightR_instruction74(void); +void CryptonightR_instruction75(void); +void CryptonightR_instruction76(void); +void CryptonightR_instruction77(void); +void CryptonightR_instruction78(void); +void CryptonightR_instruction79(void); +void CryptonightR_instruction80(void); +void CryptonightR_instruction81(void); +void CryptonightR_instruction82(void); +void CryptonightR_instruction83(void); +void CryptonightR_instruction84(void); +void CryptonightR_instruction85(void); +void CryptonightR_instruction86(void); +void CryptonightR_instruction87(void); +void CryptonightR_instruction88(void); +void CryptonightR_instruction89(void); +void CryptonightR_instruction90(void); +void CryptonightR_instruction91(void); +void CryptonightR_instruction92(void); +void CryptonightR_instruction93(void); +void CryptonightR_instruction94(void); +void CryptonightR_instruction95(void); +void CryptonightR_instruction96(void); +void CryptonightR_instruction97(void); +void CryptonightR_instruction98(void); +void CryptonightR_instruction99(void); +void CryptonightR_instruction100(void); +void CryptonightR_instruction101(void); +void CryptonightR_instruction102(void); +void CryptonightR_instruction103(void); +void CryptonightR_instruction104(void); +void CryptonightR_instruction105(void); +void CryptonightR_instruction106(void); +void CryptonightR_instruction107(void); +void CryptonightR_instruction108(void); +void CryptonightR_instruction109(void); +void CryptonightR_instruction110(void); +void CryptonightR_instruction111(void); +void CryptonightR_instruction112(void); +void CryptonightR_instruction113(void); +void CryptonightR_instruction114(void); +void CryptonightR_instruction115(void); +void CryptonightR_instruction116(void); +void CryptonightR_instruction117(void); +void CryptonightR_instruction118(void); +void CryptonightR_instruction119(void); +void CryptonightR_instruction120(void); +void CryptonightR_instruction121(void); +void CryptonightR_instruction122(void); +void CryptonightR_instruction123(void); +void CryptonightR_instruction124(void); +void CryptonightR_instruction125(void); +void CryptonightR_instruction126(void); +void CryptonightR_instruction127(void); +void CryptonightR_instruction128(void); +void CryptonightR_instruction129(void); +void CryptonightR_instruction130(void); +void CryptonightR_instruction131(void); +void CryptonightR_instruction132(void); +void CryptonightR_instruction133(void); +void CryptonightR_instruction134(void); +void CryptonightR_instruction135(void); +void CryptonightR_instruction136(void); +void CryptonightR_instruction137(void); +void CryptonightR_instruction138(void); +void CryptonightR_instruction139(void); +void CryptonightR_instruction140(void); +void CryptonightR_instruction141(void); +void CryptonightR_instruction142(void); +void CryptonightR_instruction143(void); +void CryptonightR_instruction144(void); +void CryptonightR_instruction145(void); +void CryptonightR_instruction146(void); +void CryptonightR_instruction147(void); +void CryptonightR_instruction148(void); +void CryptonightR_instruction149(void); +void CryptonightR_instruction150(void); +void CryptonightR_instruction151(void); +void CryptonightR_instruction152(void); +void CryptonightR_instruction153(void); +void CryptonightR_instruction154(void); +void CryptonightR_instruction155(void); +void CryptonightR_instruction156(void); +void CryptonightR_instruction157(void); +void CryptonightR_instruction158(void); +void CryptonightR_instruction159(void); +void CryptonightR_instruction160(void); +void CryptonightR_instruction161(void); +void CryptonightR_instruction162(void); +void CryptonightR_instruction163(void); +void CryptonightR_instruction164(void); +void CryptonightR_instruction165(void); +void CryptonightR_instruction166(void); +void CryptonightR_instruction167(void); +void CryptonightR_instruction168(void); +void CryptonightR_instruction169(void); +void CryptonightR_instruction170(void); +void CryptonightR_instruction171(void); +void CryptonightR_instruction172(void); +void CryptonightR_instruction173(void); +void CryptonightR_instruction174(void); +void CryptonightR_instruction175(void); +void CryptonightR_instruction176(void); +void CryptonightR_instruction177(void); +void CryptonightR_instruction178(void); +void CryptonightR_instruction179(void); +void CryptonightR_instruction180(void); +void CryptonightR_instruction181(void); +void CryptonightR_instruction182(void); +void CryptonightR_instruction183(void); +void CryptonightR_instruction184(void); +void CryptonightR_instruction185(void); +void CryptonightR_instruction186(void); +void CryptonightR_instruction187(void); +void CryptonightR_instruction188(void); +void CryptonightR_instruction189(void); +void CryptonightR_instruction190(void); +void CryptonightR_instruction191(void); +void CryptonightR_instruction192(void); +void CryptonightR_instruction193(void); +void CryptonightR_instruction194(void); +void CryptonightR_instruction195(void); +void CryptonightR_instruction196(void); +void CryptonightR_instruction197(void); +void CryptonightR_instruction198(void); +void CryptonightR_instruction199(void); +void CryptonightR_instruction200(void); +void CryptonightR_instruction201(void); +void CryptonightR_instruction202(void); +void CryptonightR_instruction203(void); +void CryptonightR_instruction204(void); +void CryptonightR_instruction205(void); +void CryptonightR_instruction206(void); +void CryptonightR_instruction207(void); +void CryptonightR_instruction208(void); +void CryptonightR_instruction209(void); +void CryptonightR_instruction210(void); +void CryptonightR_instruction211(void); +void CryptonightR_instruction212(void); +void CryptonightR_instruction213(void); +void CryptonightR_instruction214(void); +void CryptonightR_instruction215(void); +void CryptonightR_instruction216(void); +void CryptonightR_instruction217(void); +void CryptonightR_instruction218(void); +void CryptonightR_instruction219(void); +void CryptonightR_instruction220(void); +void CryptonightR_instruction221(void); +void CryptonightR_instruction222(void); +void CryptonightR_instruction223(void); +void CryptonightR_instruction224(void); +void CryptonightR_instruction225(void); +void CryptonightR_instruction226(void); +void CryptonightR_instruction227(void); +void CryptonightR_instruction228(void); +void CryptonightR_instruction229(void); +void CryptonightR_instruction230(void); +void CryptonightR_instruction231(void); +void CryptonightR_instruction232(void); +void CryptonightR_instruction233(void); +void CryptonightR_instruction234(void); +void CryptonightR_instruction235(void); +void CryptonightR_instruction236(void); +void CryptonightR_instruction237(void); +void CryptonightR_instruction238(void); +void CryptonightR_instruction239(void); +void CryptonightR_instruction240(void); +void CryptonightR_instruction241(void); +void CryptonightR_instruction242(void); +void CryptonightR_instruction243(void); +void CryptonightR_instruction244(void); +void CryptonightR_instruction245(void); +void CryptonightR_instruction246(void); +void CryptonightR_instruction247(void); +void CryptonightR_instruction248(void); +void CryptonightR_instruction249(void); +void CryptonightR_instruction250(void); +void CryptonightR_instruction251(void); +void CryptonightR_instruction252(void); +void CryptonightR_instruction253(void); +void CryptonightR_instruction254(void); +void CryptonightR_instruction255(void); +void CryptonightR_instruction256(void); +void CryptonightR_instruction_mov0(void); +void CryptonightR_instruction_mov1(void); +void CryptonightR_instruction_mov2(void); +void CryptonightR_instruction_mov3(void); +void CryptonightR_instruction_mov4(void); +void CryptonightR_instruction_mov5(void); +void CryptonightR_instruction_mov6(void); +void CryptonightR_instruction_mov7(void); +void CryptonightR_instruction_mov8(void); +void CryptonightR_instruction_mov9(void); +void CryptonightR_instruction_mov10(void); +void CryptonightR_instruction_mov11(void); +void CryptonightR_instruction_mov12(void); +void CryptonightR_instruction_mov13(void); +void CryptonightR_instruction_mov14(void); +void CryptonightR_instruction_mov15(void); +void CryptonightR_instruction_mov16(void); +void CryptonightR_instruction_mov17(void); +void CryptonightR_instruction_mov18(void); +void CryptonightR_instruction_mov19(void); +void CryptonightR_instruction_mov20(void); +void CryptonightR_instruction_mov21(void); +void CryptonightR_instruction_mov22(void); +void CryptonightR_instruction_mov23(void); +void CryptonightR_instruction_mov24(void); +void CryptonightR_instruction_mov25(void); +void CryptonightR_instruction_mov26(void); +void CryptonightR_instruction_mov27(void); +void CryptonightR_instruction_mov28(void); +void CryptonightR_instruction_mov29(void); +void CryptonightR_instruction_mov30(void); +void CryptonightR_instruction_mov31(void); +void CryptonightR_instruction_mov32(void); +void CryptonightR_instruction_mov33(void); +void CryptonightR_instruction_mov34(void); +void CryptonightR_instruction_mov35(void); +void CryptonightR_instruction_mov36(void); +void CryptonightR_instruction_mov37(void); +void CryptonightR_instruction_mov38(void); +void CryptonightR_instruction_mov39(void); +void CryptonightR_instruction_mov40(void); +void CryptonightR_instruction_mov41(void); +void CryptonightR_instruction_mov42(void); +void CryptonightR_instruction_mov43(void); +void CryptonightR_instruction_mov44(void); +void CryptonightR_instruction_mov45(void); +void CryptonightR_instruction_mov46(void); +void CryptonightR_instruction_mov47(void); +void CryptonightR_instruction_mov48(void); +void CryptonightR_instruction_mov49(void); +void CryptonightR_instruction_mov50(void); +void CryptonightR_instruction_mov51(void); +void CryptonightR_instruction_mov52(void); +void CryptonightR_instruction_mov53(void); +void CryptonightR_instruction_mov54(void); +void CryptonightR_instruction_mov55(void); +void CryptonightR_instruction_mov56(void); +void CryptonightR_instruction_mov57(void); +void CryptonightR_instruction_mov58(void); +void CryptonightR_instruction_mov59(void); +void CryptonightR_instruction_mov60(void); +void CryptonightR_instruction_mov61(void); +void CryptonightR_instruction_mov62(void); +void CryptonightR_instruction_mov63(void); +void CryptonightR_instruction_mov64(void); +void CryptonightR_instruction_mov65(void); +void CryptonightR_instruction_mov66(void); +void CryptonightR_instruction_mov67(void); +void CryptonightR_instruction_mov68(void); +void CryptonightR_instruction_mov69(void); +void CryptonightR_instruction_mov70(void); +void CryptonightR_instruction_mov71(void); +void CryptonightR_instruction_mov72(void); +void CryptonightR_instruction_mov73(void); +void CryptonightR_instruction_mov74(void); +void CryptonightR_instruction_mov75(void); +void CryptonightR_instruction_mov76(void); +void CryptonightR_instruction_mov77(void); +void CryptonightR_instruction_mov78(void); +void CryptonightR_instruction_mov79(void); +void CryptonightR_instruction_mov80(void); +void CryptonightR_instruction_mov81(void); +void CryptonightR_instruction_mov82(void); +void CryptonightR_instruction_mov83(void); +void CryptonightR_instruction_mov84(void); +void CryptonightR_instruction_mov85(void); +void CryptonightR_instruction_mov86(void); +void CryptonightR_instruction_mov87(void); +void CryptonightR_instruction_mov88(void); +void CryptonightR_instruction_mov89(void); +void CryptonightR_instruction_mov90(void); +void CryptonightR_instruction_mov91(void); +void CryptonightR_instruction_mov92(void); +void CryptonightR_instruction_mov93(void); +void CryptonightR_instruction_mov94(void); +void CryptonightR_instruction_mov95(void); +void CryptonightR_instruction_mov96(void); +void CryptonightR_instruction_mov97(void); +void CryptonightR_instruction_mov98(void); +void CryptonightR_instruction_mov99(void); +void CryptonightR_instruction_mov100(void); +void CryptonightR_instruction_mov101(void); +void CryptonightR_instruction_mov102(void); +void CryptonightR_instruction_mov103(void); +void CryptonightR_instruction_mov104(void); +void CryptonightR_instruction_mov105(void); +void CryptonightR_instruction_mov106(void); +void CryptonightR_instruction_mov107(void); +void CryptonightR_instruction_mov108(void); +void CryptonightR_instruction_mov109(void); +void CryptonightR_instruction_mov110(void); +void CryptonightR_instruction_mov111(void); +void CryptonightR_instruction_mov112(void); +void CryptonightR_instruction_mov113(void); +void CryptonightR_instruction_mov114(void); +void CryptonightR_instruction_mov115(void); +void CryptonightR_instruction_mov116(void); +void CryptonightR_instruction_mov117(void); +void CryptonightR_instruction_mov118(void); +void CryptonightR_instruction_mov119(void); +void CryptonightR_instruction_mov120(void); +void CryptonightR_instruction_mov121(void); +void CryptonightR_instruction_mov122(void); +void CryptonightR_instruction_mov123(void); +void CryptonightR_instruction_mov124(void); +void CryptonightR_instruction_mov125(void); +void CryptonightR_instruction_mov126(void); +void CryptonightR_instruction_mov127(void); +void CryptonightR_instruction_mov128(void); +void CryptonightR_instruction_mov129(void); +void CryptonightR_instruction_mov130(void); +void CryptonightR_instruction_mov131(void); +void CryptonightR_instruction_mov132(void); +void CryptonightR_instruction_mov133(void); +void CryptonightR_instruction_mov134(void); +void CryptonightR_instruction_mov135(void); +void CryptonightR_instruction_mov136(void); +void CryptonightR_instruction_mov137(void); +void CryptonightR_instruction_mov138(void); +void CryptonightR_instruction_mov139(void); +void CryptonightR_instruction_mov140(void); +void CryptonightR_instruction_mov141(void); +void CryptonightR_instruction_mov142(void); +void CryptonightR_instruction_mov143(void); +void CryptonightR_instruction_mov144(void); +void CryptonightR_instruction_mov145(void); +void CryptonightR_instruction_mov146(void); +void CryptonightR_instruction_mov147(void); +void CryptonightR_instruction_mov148(void); +void CryptonightR_instruction_mov149(void); +void CryptonightR_instruction_mov150(void); +void CryptonightR_instruction_mov151(void); +void CryptonightR_instruction_mov152(void); +void CryptonightR_instruction_mov153(void); +void CryptonightR_instruction_mov154(void); +void CryptonightR_instruction_mov155(void); +void CryptonightR_instruction_mov156(void); +void CryptonightR_instruction_mov157(void); +void CryptonightR_instruction_mov158(void); +void CryptonightR_instruction_mov159(void); +void CryptonightR_instruction_mov160(void); +void CryptonightR_instruction_mov161(void); +void CryptonightR_instruction_mov162(void); +void CryptonightR_instruction_mov163(void); +void CryptonightR_instruction_mov164(void); +void CryptonightR_instruction_mov165(void); +void CryptonightR_instruction_mov166(void); +void CryptonightR_instruction_mov167(void); +void CryptonightR_instruction_mov168(void); +void CryptonightR_instruction_mov169(void); +void CryptonightR_instruction_mov170(void); +void CryptonightR_instruction_mov171(void); +void CryptonightR_instruction_mov172(void); +void CryptonightR_instruction_mov173(void); +void CryptonightR_instruction_mov174(void); +void CryptonightR_instruction_mov175(void); +void CryptonightR_instruction_mov176(void); +void CryptonightR_instruction_mov177(void); +void CryptonightR_instruction_mov178(void); +void CryptonightR_instruction_mov179(void); +void CryptonightR_instruction_mov180(void); +void CryptonightR_instruction_mov181(void); +void CryptonightR_instruction_mov182(void); +void CryptonightR_instruction_mov183(void); +void CryptonightR_instruction_mov184(void); +void CryptonightR_instruction_mov185(void); +void CryptonightR_instruction_mov186(void); +void CryptonightR_instruction_mov187(void); +void CryptonightR_instruction_mov188(void); +void CryptonightR_instruction_mov189(void); +void CryptonightR_instruction_mov190(void); +void CryptonightR_instruction_mov191(void); +void CryptonightR_instruction_mov192(void); +void CryptonightR_instruction_mov193(void); +void CryptonightR_instruction_mov194(void); +void CryptonightR_instruction_mov195(void); +void CryptonightR_instruction_mov196(void); +void CryptonightR_instruction_mov197(void); +void CryptonightR_instruction_mov198(void); +void CryptonightR_instruction_mov199(void); +void CryptonightR_instruction_mov200(void); +void CryptonightR_instruction_mov201(void); +void CryptonightR_instruction_mov202(void); +void CryptonightR_instruction_mov203(void); +void CryptonightR_instruction_mov204(void); +void CryptonightR_instruction_mov205(void); +void CryptonightR_instruction_mov206(void); +void CryptonightR_instruction_mov207(void); +void CryptonightR_instruction_mov208(void); +void CryptonightR_instruction_mov209(void); +void CryptonightR_instruction_mov210(void); +void CryptonightR_instruction_mov211(void); +void CryptonightR_instruction_mov212(void); +void CryptonightR_instruction_mov213(void); +void CryptonightR_instruction_mov214(void); +void CryptonightR_instruction_mov215(void); +void CryptonightR_instruction_mov216(void); +void CryptonightR_instruction_mov217(void); +void CryptonightR_instruction_mov218(void); +void CryptonightR_instruction_mov219(void); +void CryptonightR_instruction_mov220(void); +void CryptonightR_instruction_mov221(void); +void CryptonightR_instruction_mov222(void); +void CryptonightR_instruction_mov223(void); +void CryptonightR_instruction_mov224(void); +void CryptonightR_instruction_mov225(void); +void CryptonightR_instruction_mov226(void); +void CryptonightR_instruction_mov227(void); +void CryptonightR_instruction_mov228(void); +void CryptonightR_instruction_mov229(void); +void CryptonightR_instruction_mov230(void); +void CryptonightR_instruction_mov231(void); +void CryptonightR_instruction_mov232(void); +void CryptonightR_instruction_mov233(void); +void CryptonightR_instruction_mov234(void); +void CryptonightR_instruction_mov235(void); +void CryptonightR_instruction_mov236(void); +void CryptonightR_instruction_mov237(void); +void CryptonightR_instruction_mov238(void); +void CryptonightR_instruction_mov239(void); +void CryptonightR_instruction_mov240(void); +void CryptonightR_instruction_mov241(void); +void CryptonightR_instruction_mov242(void); +void CryptonightR_instruction_mov243(void); +void CryptonightR_instruction_mov244(void); +void CryptonightR_instruction_mov245(void); +void CryptonightR_instruction_mov246(void); +void CryptonightR_instruction_mov247(void); +void CryptonightR_instruction_mov248(void); +void CryptonightR_instruction_mov249(void); +void CryptonightR_instruction_mov250(void); +void CryptonightR_instruction_mov251(void); +void CryptonightR_instruction_mov252(void); +void CryptonightR_instruction_mov253(void); +void CryptonightR_instruction_mov254(void); +void CryptonightR_instruction_mov255(void); +void CryptonightR_instruction_mov256(void); + +const void* instructions[257] = { + CryptonightR_instruction0, + CryptonightR_instruction1, + CryptonightR_instruction2, + CryptonightR_instruction3, + CryptonightR_instruction4, + CryptonightR_instruction5, + CryptonightR_instruction6, + CryptonightR_instruction7, + CryptonightR_instruction8, + CryptonightR_instruction9, + CryptonightR_instruction10, + CryptonightR_instruction11, + CryptonightR_instruction12, + CryptonightR_instruction13, + CryptonightR_instruction14, + CryptonightR_instruction15, + CryptonightR_instruction16, + CryptonightR_instruction17, + CryptonightR_instruction18, + CryptonightR_instruction19, + CryptonightR_instruction20, + CryptonightR_instruction21, + CryptonightR_instruction22, + CryptonightR_instruction23, + CryptonightR_instruction24, + CryptonightR_instruction25, + CryptonightR_instruction26, + CryptonightR_instruction27, + CryptonightR_instruction28, + CryptonightR_instruction29, + CryptonightR_instruction30, + CryptonightR_instruction31, + CryptonightR_instruction32, + CryptonightR_instruction33, + CryptonightR_instruction34, + CryptonightR_instruction35, + CryptonightR_instruction36, + CryptonightR_instruction37, + CryptonightR_instruction38, + CryptonightR_instruction39, + CryptonightR_instruction40, + CryptonightR_instruction41, + CryptonightR_instruction42, + CryptonightR_instruction43, + CryptonightR_instruction44, + CryptonightR_instruction45, + CryptonightR_instruction46, + CryptonightR_instruction47, + CryptonightR_instruction48, + CryptonightR_instruction49, + CryptonightR_instruction50, + CryptonightR_instruction51, + CryptonightR_instruction52, + CryptonightR_instruction53, + CryptonightR_instruction54, + CryptonightR_instruction55, + CryptonightR_instruction56, + CryptonightR_instruction57, + CryptonightR_instruction58, + CryptonightR_instruction59, + CryptonightR_instruction60, + CryptonightR_instruction61, + CryptonightR_instruction62, + CryptonightR_instruction63, + CryptonightR_instruction64, + CryptonightR_instruction65, + CryptonightR_instruction66, + CryptonightR_instruction67, + CryptonightR_instruction68, + CryptonightR_instruction69, + CryptonightR_instruction70, + CryptonightR_instruction71, + CryptonightR_instruction72, + CryptonightR_instruction73, + CryptonightR_instruction74, + CryptonightR_instruction75, + CryptonightR_instruction76, + CryptonightR_instruction77, + CryptonightR_instruction78, + CryptonightR_instruction79, + CryptonightR_instruction80, + CryptonightR_instruction81, + CryptonightR_instruction82, + CryptonightR_instruction83, + CryptonightR_instruction84, + CryptonightR_instruction85, + CryptonightR_instruction86, + CryptonightR_instruction87, + CryptonightR_instruction88, + CryptonightR_instruction89, + CryptonightR_instruction90, + CryptonightR_instruction91, + CryptonightR_instruction92, + CryptonightR_instruction93, + CryptonightR_instruction94, + CryptonightR_instruction95, + CryptonightR_instruction96, + CryptonightR_instruction97, + CryptonightR_instruction98, + CryptonightR_instruction99, + CryptonightR_instruction100, + CryptonightR_instruction101, + CryptonightR_instruction102, + CryptonightR_instruction103, + CryptonightR_instruction104, + CryptonightR_instruction105, + CryptonightR_instruction106, + CryptonightR_instruction107, + CryptonightR_instruction108, + CryptonightR_instruction109, + CryptonightR_instruction110, + CryptonightR_instruction111, + CryptonightR_instruction112, + CryptonightR_instruction113, + CryptonightR_instruction114, + CryptonightR_instruction115, + CryptonightR_instruction116, + CryptonightR_instruction117, + CryptonightR_instruction118, + CryptonightR_instruction119, + CryptonightR_instruction120, + CryptonightR_instruction121, + CryptonightR_instruction122, + CryptonightR_instruction123, + CryptonightR_instruction124, + CryptonightR_instruction125, + CryptonightR_instruction126, + CryptonightR_instruction127, + CryptonightR_instruction128, + CryptonightR_instruction129, + CryptonightR_instruction130, + CryptonightR_instruction131, + CryptonightR_instruction132, + CryptonightR_instruction133, + CryptonightR_instruction134, + CryptonightR_instruction135, + CryptonightR_instruction136, + CryptonightR_instruction137, + CryptonightR_instruction138, + CryptonightR_instruction139, + CryptonightR_instruction140, + CryptonightR_instruction141, + CryptonightR_instruction142, + CryptonightR_instruction143, + CryptonightR_instruction144, + CryptonightR_instruction145, + CryptonightR_instruction146, + CryptonightR_instruction147, + CryptonightR_instruction148, + CryptonightR_instruction149, + CryptonightR_instruction150, + CryptonightR_instruction151, + CryptonightR_instruction152, + CryptonightR_instruction153, + CryptonightR_instruction154, + CryptonightR_instruction155, + CryptonightR_instruction156, + CryptonightR_instruction157, + CryptonightR_instruction158, + CryptonightR_instruction159, + CryptonightR_instruction160, + CryptonightR_instruction161, + CryptonightR_instruction162, + CryptonightR_instruction163, + CryptonightR_instruction164, + CryptonightR_instruction165, + CryptonightR_instruction166, + CryptonightR_instruction167, + CryptonightR_instruction168, + CryptonightR_instruction169, + CryptonightR_instruction170, + CryptonightR_instruction171, + CryptonightR_instruction172, + CryptonightR_instruction173, + CryptonightR_instruction174, + CryptonightR_instruction175, + CryptonightR_instruction176, + CryptonightR_instruction177, + CryptonightR_instruction178, + CryptonightR_instruction179, + CryptonightR_instruction180, + CryptonightR_instruction181, + CryptonightR_instruction182, + CryptonightR_instruction183, + CryptonightR_instruction184, + CryptonightR_instruction185, + CryptonightR_instruction186, + CryptonightR_instruction187, + CryptonightR_instruction188, + CryptonightR_instruction189, + CryptonightR_instruction190, + CryptonightR_instruction191, + CryptonightR_instruction192, + CryptonightR_instruction193, + CryptonightR_instruction194, + CryptonightR_instruction195, + CryptonightR_instruction196, + CryptonightR_instruction197, + CryptonightR_instruction198, + CryptonightR_instruction199, + CryptonightR_instruction200, + CryptonightR_instruction201, + CryptonightR_instruction202, + CryptonightR_instruction203, + CryptonightR_instruction204, + CryptonightR_instruction205, + CryptonightR_instruction206, + CryptonightR_instruction207, + CryptonightR_instruction208, + CryptonightR_instruction209, + CryptonightR_instruction210, + CryptonightR_instruction211, + CryptonightR_instruction212, + CryptonightR_instruction213, + CryptonightR_instruction214, + CryptonightR_instruction215, + CryptonightR_instruction216, + CryptonightR_instruction217, + CryptonightR_instruction218, + CryptonightR_instruction219, + CryptonightR_instruction220, + CryptonightR_instruction221, + CryptonightR_instruction222, + CryptonightR_instruction223, + CryptonightR_instruction224, + CryptonightR_instruction225, + CryptonightR_instruction226, + CryptonightR_instruction227, + CryptonightR_instruction228, + CryptonightR_instruction229, + CryptonightR_instruction230, + CryptonightR_instruction231, + CryptonightR_instruction232, + CryptonightR_instruction233, + CryptonightR_instruction234, + CryptonightR_instruction235, + CryptonightR_instruction236, + CryptonightR_instruction237, + CryptonightR_instruction238, + CryptonightR_instruction239, + CryptonightR_instruction240, + CryptonightR_instruction241, + CryptonightR_instruction242, + CryptonightR_instruction243, + CryptonightR_instruction244, + CryptonightR_instruction245, + CryptonightR_instruction246, + CryptonightR_instruction247, + CryptonightR_instruction248, + CryptonightR_instruction249, + CryptonightR_instruction250, + CryptonightR_instruction251, + CryptonightR_instruction252, + CryptonightR_instruction253, + CryptonightR_instruction254, + CryptonightR_instruction255, + CryptonightR_instruction256, +}; + +const void* instructions_mov[257] = { + CryptonightR_instruction_mov0, + CryptonightR_instruction_mov1, + CryptonightR_instruction_mov2, + CryptonightR_instruction_mov3, + CryptonightR_instruction_mov4, + CryptonightR_instruction_mov5, + CryptonightR_instruction_mov6, + CryptonightR_instruction_mov7, + CryptonightR_instruction_mov8, + CryptonightR_instruction_mov9, + CryptonightR_instruction_mov10, + CryptonightR_instruction_mov11, + CryptonightR_instruction_mov12, + CryptonightR_instruction_mov13, + CryptonightR_instruction_mov14, + CryptonightR_instruction_mov15, + CryptonightR_instruction_mov16, + CryptonightR_instruction_mov17, + CryptonightR_instruction_mov18, + CryptonightR_instruction_mov19, + CryptonightR_instruction_mov20, + CryptonightR_instruction_mov21, + CryptonightR_instruction_mov22, + CryptonightR_instruction_mov23, + CryptonightR_instruction_mov24, + CryptonightR_instruction_mov25, + CryptonightR_instruction_mov26, + CryptonightR_instruction_mov27, + CryptonightR_instruction_mov28, + CryptonightR_instruction_mov29, + CryptonightR_instruction_mov30, + CryptonightR_instruction_mov31, + CryptonightR_instruction_mov32, + CryptonightR_instruction_mov33, + CryptonightR_instruction_mov34, + CryptonightR_instruction_mov35, + CryptonightR_instruction_mov36, + CryptonightR_instruction_mov37, + CryptonightR_instruction_mov38, + CryptonightR_instruction_mov39, + CryptonightR_instruction_mov40, + CryptonightR_instruction_mov41, + CryptonightR_instruction_mov42, + CryptonightR_instruction_mov43, + CryptonightR_instruction_mov44, + CryptonightR_instruction_mov45, + CryptonightR_instruction_mov46, + CryptonightR_instruction_mov47, + CryptonightR_instruction_mov48, + CryptonightR_instruction_mov49, + CryptonightR_instruction_mov50, + CryptonightR_instruction_mov51, + CryptonightR_instruction_mov52, + CryptonightR_instruction_mov53, + CryptonightR_instruction_mov54, + CryptonightR_instruction_mov55, + CryptonightR_instruction_mov56, + CryptonightR_instruction_mov57, + CryptonightR_instruction_mov58, + CryptonightR_instruction_mov59, + CryptonightR_instruction_mov60, + CryptonightR_instruction_mov61, + CryptonightR_instruction_mov62, + CryptonightR_instruction_mov63, + CryptonightR_instruction_mov64, + CryptonightR_instruction_mov65, + CryptonightR_instruction_mov66, + CryptonightR_instruction_mov67, + CryptonightR_instruction_mov68, + CryptonightR_instruction_mov69, + CryptonightR_instruction_mov70, + CryptonightR_instruction_mov71, + CryptonightR_instruction_mov72, + CryptonightR_instruction_mov73, + CryptonightR_instruction_mov74, + CryptonightR_instruction_mov75, + CryptonightR_instruction_mov76, + CryptonightR_instruction_mov77, + CryptonightR_instruction_mov78, + CryptonightR_instruction_mov79, + CryptonightR_instruction_mov80, + CryptonightR_instruction_mov81, + CryptonightR_instruction_mov82, + CryptonightR_instruction_mov83, + CryptonightR_instruction_mov84, + CryptonightR_instruction_mov85, + CryptonightR_instruction_mov86, + CryptonightR_instruction_mov87, + CryptonightR_instruction_mov88, + CryptonightR_instruction_mov89, + CryptonightR_instruction_mov90, + CryptonightR_instruction_mov91, + CryptonightR_instruction_mov92, + CryptonightR_instruction_mov93, + CryptonightR_instruction_mov94, + CryptonightR_instruction_mov95, + CryptonightR_instruction_mov96, + CryptonightR_instruction_mov97, + CryptonightR_instruction_mov98, + CryptonightR_instruction_mov99, + CryptonightR_instruction_mov100, + CryptonightR_instruction_mov101, + CryptonightR_instruction_mov102, + CryptonightR_instruction_mov103, + CryptonightR_instruction_mov104, + CryptonightR_instruction_mov105, + CryptonightR_instruction_mov106, + CryptonightR_instruction_mov107, + CryptonightR_instruction_mov108, + CryptonightR_instruction_mov109, + CryptonightR_instruction_mov110, + CryptonightR_instruction_mov111, + CryptonightR_instruction_mov112, + CryptonightR_instruction_mov113, + CryptonightR_instruction_mov114, + CryptonightR_instruction_mov115, + CryptonightR_instruction_mov116, + CryptonightR_instruction_mov117, + CryptonightR_instruction_mov118, + CryptonightR_instruction_mov119, + CryptonightR_instruction_mov120, + CryptonightR_instruction_mov121, + CryptonightR_instruction_mov122, + CryptonightR_instruction_mov123, + CryptonightR_instruction_mov124, + CryptonightR_instruction_mov125, + CryptonightR_instruction_mov126, + CryptonightR_instruction_mov127, + CryptonightR_instruction_mov128, + CryptonightR_instruction_mov129, + CryptonightR_instruction_mov130, + CryptonightR_instruction_mov131, + CryptonightR_instruction_mov132, + CryptonightR_instruction_mov133, + CryptonightR_instruction_mov134, + CryptonightR_instruction_mov135, + CryptonightR_instruction_mov136, + CryptonightR_instruction_mov137, + CryptonightR_instruction_mov138, + CryptonightR_instruction_mov139, + CryptonightR_instruction_mov140, + CryptonightR_instruction_mov141, + CryptonightR_instruction_mov142, + CryptonightR_instruction_mov143, + CryptonightR_instruction_mov144, + CryptonightR_instruction_mov145, + CryptonightR_instruction_mov146, + CryptonightR_instruction_mov147, + CryptonightR_instruction_mov148, + CryptonightR_instruction_mov149, + CryptonightR_instruction_mov150, + CryptonightR_instruction_mov151, + CryptonightR_instruction_mov152, + CryptonightR_instruction_mov153, + CryptonightR_instruction_mov154, + CryptonightR_instruction_mov155, + CryptonightR_instruction_mov156, + CryptonightR_instruction_mov157, + CryptonightR_instruction_mov158, + CryptonightR_instruction_mov159, + CryptonightR_instruction_mov160, + CryptonightR_instruction_mov161, + CryptonightR_instruction_mov162, + CryptonightR_instruction_mov163, + CryptonightR_instruction_mov164, + CryptonightR_instruction_mov165, + CryptonightR_instruction_mov166, + CryptonightR_instruction_mov167, + CryptonightR_instruction_mov168, + CryptonightR_instruction_mov169, + CryptonightR_instruction_mov170, + CryptonightR_instruction_mov171, + CryptonightR_instruction_mov172, + CryptonightR_instruction_mov173, + CryptonightR_instruction_mov174, + CryptonightR_instruction_mov175, + CryptonightR_instruction_mov176, + CryptonightR_instruction_mov177, + CryptonightR_instruction_mov178, + CryptonightR_instruction_mov179, + CryptonightR_instruction_mov180, + CryptonightR_instruction_mov181, + CryptonightR_instruction_mov182, + CryptonightR_instruction_mov183, + CryptonightR_instruction_mov184, + CryptonightR_instruction_mov185, + CryptonightR_instruction_mov186, + CryptonightR_instruction_mov187, + CryptonightR_instruction_mov188, + CryptonightR_instruction_mov189, + CryptonightR_instruction_mov190, + CryptonightR_instruction_mov191, + CryptonightR_instruction_mov192, + CryptonightR_instruction_mov193, + CryptonightR_instruction_mov194, + CryptonightR_instruction_mov195, + CryptonightR_instruction_mov196, + CryptonightR_instruction_mov197, + CryptonightR_instruction_mov198, + CryptonightR_instruction_mov199, + CryptonightR_instruction_mov200, + CryptonightR_instruction_mov201, + CryptonightR_instruction_mov202, + CryptonightR_instruction_mov203, + CryptonightR_instruction_mov204, + CryptonightR_instruction_mov205, + CryptonightR_instruction_mov206, + CryptonightR_instruction_mov207, + CryptonightR_instruction_mov208, + CryptonightR_instruction_mov209, + CryptonightR_instruction_mov210, + CryptonightR_instruction_mov211, + CryptonightR_instruction_mov212, + CryptonightR_instruction_mov213, + CryptonightR_instruction_mov214, + CryptonightR_instruction_mov215, + CryptonightR_instruction_mov216, + CryptonightR_instruction_mov217, + CryptonightR_instruction_mov218, + CryptonightR_instruction_mov219, + CryptonightR_instruction_mov220, + CryptonightR_instruction_mov221, + CryptonightR_instruction_mov222, + CryptonightR_instruction_mov223, + CryptonightR_instruction_mov224, + CryptonightR_instruction_mov225, + CryptonightR_instruction_mov226, + CryptonightR_instruction_mov227, + CryptonightR_instruction_mov228, + CryptonightR_instruction_mov229, + CryptonightR_instruction_mov230, + CryptonightR_instruction_mov231, + CryptonightR_instruction_mov232, + CryptonightR_instruction_mov233, + CryptonightR_instruction_mov234, + CryptonightR_instruction_mov235, + CryptonightR_instruction_mov236, + CryptonightR_instruction_mov237, + CryptonightR_instruction_mov238, + CryptonightR_instruction_mov239, + CryptonightR_instruction_mov240, + CryptonightR_instruction_mov241, + CryptonightR_instruction_mov242, + CryptonightR_instruction_mov243, + CryptonightR_instruction_mov244, + CryptonightR_instruction_mov245, + CryptonightR_instruction_mov246, + CryptonightR_instruction_mov247, + CryptonightR_instruction_mov248, + CryptonightR_instruction_mov249, + CryptonightR_instruction_mov250, + CryptonightR_instruction_mov251, + CryptonightR_instruction_mov252, + CryptonightR_instruction_mov253, + CryptonightR_instruction_mov254, + CryptonightR_instruction_mov255, + CryptonightR_instruction_mov256, +}; + +#endif + +#endif // CRYPTONIGHTR_TEMPLATE_H diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index 6ef7d4207..1e305b3a6 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/blake256.h b/src/crypto/blake256.h index 073772289..309dbf3ec 100644 --- a/src/crypto/blake256.h +++ b/src/crypto/blake256.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/chacha.h b/src/crypto/chacha.h index 6e85ad0e9..a39823e5a 100644 --- a/src/crypto/chacha.h +++ b/src/crypto/chacha.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -73,18 +73,18 @@ namespace crypto { inline void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; - crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); + crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 0/*prehashed*/, 0/*height*/); for (uint64_t n = 1; n < kdf_rounds; ++n) - crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/, 0/*height*/); memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; - crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 1/*prehashed*/); + crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 1/*prehashed*/, 0/*height*/); for (uint64_t n = 1; n < kdf_rounds; ++n) - crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/, 0/*height*/); memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 1f77513ca..c9530bb2a 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 09296d6f9..5a3d994a6 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index 2910dafd4..7137437bc 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index ddf072f68..3f06c4f3f 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index f22df1230..22b182ab0 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/README.md b/src/crypto/crypto_ops_builder/README.md index 326d2ca6e..4bb95cc4a 100644 --- a/src/crypto/crypto_ops_builder/README.md +++ b/src/crypto/crypto_ops_builder/README.md @@ -1,6 +1,6 @@ # Monero -Copyright (c) 2014-2018, The Monero Project +Copyright (c) 2014-2019, The Monero Project ## Crypto Ops Builder diff --git a/src/crypto/crypto_ops_builder/crypto-ops-data.c b/src/crypto/crypto_ops_builder/crypto-ops-data.c index 127e3e17b..45e9923b1 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops-data.c +++ b/src/crypto/crypto_ops_builder/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/crypto-ops-old.c b/src/crypto/crypto_ops_builder/crypto-ops-old.c index 9097bf95b..89c2ced6e 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops-old.c +++ b/src/crypto/crypto_ops_builder/crypto-ops-old.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/crypto-ops.h b/src/crypto/crypto_ops_builder/crypto-ops.h index 9337b56b7..b4fcfca9c 100644 --- a/src/crypto/crypto_ops_builder/crypto-ops.h +++ b/src/crypto/crypto_ops_builder/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py index 0ed97d5f4..dfba583f7 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py @@ -15,7 +15,7 @@ print("maybe someone smart can replace the sed with perl..") a = "" license = textwrap.dedent("""\ - // Copyright (c) 2014-2018, The Monero Project + // Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h b/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h index c06af035f..f62ff441d 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 42b98706e..d06726638 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl.h b/src/crypto/groestl.h index 19837f309..6628947dd 100644 --- a/src/crypto/groestl.h +++ b/src/crypto/groestl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index 12472dced..ca0c4fca6 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-blake.c b/src/crypto/hash-extra-blake.c index d33103c97..9bada96f3 100644 --- a/src/crypto/hash-extra-blake.c +++ b/src/crypto/hash-extra-blake.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-groestl.c b/src/crypto/hash-extra-groestl.c index 228853a44..57866bf9d 100644 --- a/src/crypto/hash-extra-groestl.c +++ b/src/crypto/hash-extra-groestl.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-jh.c b/src/crypto/hash-extra-jh.c index e765a18f3..0dbac4fb5 100644 --- a/src/crypto/hash-extra-jh.c +++ b/src/crypto/hash-extra-jh.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-skein.c b/src/crypto/hash-extra-skein.c index 06d8f87cc..78f48609f 100644 --- a/src/crypto/hash-extra-skein.c +++ b/src/crypto/hash-extra-skein.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index 77b52e2d4..859c810bd 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -79,7 +79,7 @@ enum { }; void cn_fast_hash(const void *data, size_t length, char *hash); -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed); +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height); void hash_extra_blake(const void *data, size_t length, char *hash); void hash_extra_groestl(const void *data, size_t length, char *hash); diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 43ce32957..b66f3b010 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 995e2294e..17071923d 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -71,12 +71,12 @@ namespace crypto { return h; } - inline void cn_slow_hash(const void *data, std::size_t length, hash &hash, int variant = 0) { - cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 0/*prehashed*/); + inline void cn_slow_hash(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) { + cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 0/*prehashed*/, height); } - inline void cn_slow_hash_prehashed(const void *data, std::size_t length, hash &hash, int variant = 0) { - cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 1/*prehashed*/); + inline void cn_slow_hash_prehashed(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) { + cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 1/*prehashed*/, height); } inline void tree_hash(const hash *hashes, std::size_t count, hash &root_hash) { diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index 107988d2b..75d80f054 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.c b/src/crypto/random.c index 9e1a70a2d..74b202661 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.h b/src/crypto/random.h index 6468136cc..ccb9f4853 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/skein_port.h b/src/crypto/skein_port.h index 8a1640e57..1ec07a4d1 100644 --- a/src/crypto/skein_port.h +++ b/src/crypto/skein_port.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index ae0bd4e98..9dc6f5038 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -39,6 +39,11 @@ #include "hash-ops.h" #include "oaes_lib.h" #include "variant2_int_sqrt.h" +#include "variant4_random_math.h" +#include "CryptonightR_JIT.h" + +#include <errno.h> +#include <string.h> #define MEMORY (1 << 21) // 2MB scratchpad #define ITER (1 << 20) @@ -50,6 +55,41 @@ extern void aesb_single_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +volatile int use_v4_jit_flag = -1; + +static inline int use_v4_jit(void) +{ +#if defined(__x86_64__) + + if (use_v4_jit_flag != -1) + return use_v4_jit_flag; + + const char *env = getenv("MONERO_USE_CNV4_JIT"); + if (!env) { + use_v4_jit_flag = 0; + } + else if (!strcmp(env, "0") || !strcmp(env, "no")) { + use_v4_jit_flag = 0; + } + else { + use_v4_jit_flag = 1; + } + return use_v4_jit_flag; +#else + return 0; +#endif +} + #define VARIANT1_1(p) \ do if (variant == 1) \ { \ @@ -116,48 +156,74 @@ extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *ex #define VARIANT2_SHUFFLE_ADD_SSE2(base_ptr, offset) \ do if (variant >= 2) \ { \ - const __m128i chunk1 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10))); \ + __m128i chunk1 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10))); \ const __m128i chunk2 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20))); \ const __m128i chunk3 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30))); \ _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10)), _mm_add_epi64(chunk3, _b1)); \ _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20)), _mm_add_epi64(chunk1, _b)); \ _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30)), _mm_add_epi64(chunk2, _a)); \ + if (variant >= 4) \ + { \ + chunk1 = _mm_xor_si128(chunk1, chunk2); \ + _c = _mm_xor_si128(_c, chunk3); \ + _c = _mm_xor_si128(_c, chunk1); \ + } \ } while (0) #define VARIANT2_SHUFFLE_ADD_NEON(base_ptr, offset) \ do if (variant >= 2) \ { \ - const uint64x2_t chunk1 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x10))); \ + uint64x2_t chunk1 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x10))); \ const uint64x2_t chunk2 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x20))); \ const uint64x2_t chunk3 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x30))); \ vst1q_u64(U64((base_ptr) + ((offset) ^ 0x10)), vaddq_u64(chunk3, vreinterpretq_u64_u8(_b1))); \ vst1q_u64(U64((base_ptr) + ((offset) ^ 0x20)), vaddq_u64(chunk1, vreinterpretq_u64_u8(_b))); \ vst1q_u64(U64((base_ptr) + ((offset) ^ 0x30)), vaddq_u64(chunk2, vreinterpretq_u64_u8(_a))); \ + if (variant >= 4) \ + { \ + chunk1 = veorq_u64(chunk1, chunk2); \ + _c = vreinterpretq_u8_u64(veorq_u64(vreinterpretq_u64_u8(_c), chunk3)); \ + _c = vreinterpretq_u8_u64(veorq_u64(vreinterpretq_u64_u8(_c), chunk1)); \ + } \ } while (0) -#define VARIANT2_PORTABLE_SHUFFLE_ADD(base_ptr, offset) \ +#define VARIANT2_PORTABLE_SHUFFLE_ADD(out, a_, base_ptr, offset) \ do if (variant >= 2) \ { \ uint64_t* chunk1 = U64((base_ptr) + ((offset) ^ 0x10)); \ uint64_t* chunk2 = U64((base_ptr) + ((offset) ^ 0x20)); \ uint64_t* chunk3 = U64((base_ptr) + ((offset) ^ 0x30)); \ \ - const uint64_t chunk1_old[2] = { chunk1[0], chunk1[1] }; \ + uint64_t chunk1_old[2] = { SWAP64LE(chunk1[0]), SWAP64LE(chunk1[1]) }; \ + const uint64_t chunk2_old[2] = { SWAP64LE(chunk2[0]), SWAP64LE(chunk2[1]) }; \ + const uint64_t chunk3_old[2] = { SWAP64LE(chunk3[0]), SWAP64LE(chunk3[1]) }; \ \ uint64_t b1[2]; \ memcpy_swap64le(b1, b + 16, 2); \ - chunk1[0] = SWAP64LE(SWAP64LE(chunk3[0]) + b1[0]); \ - chunk1[1] = SWAP64LE(SWAP64LE(chunk3[1]) + b1[1]); \ + chunk1[0] = SWAP64LE(chunk3_old[0] + b1[0]); \ + chunk1[1] = SWAP64LE(chunk3_old[1] + b1[1]); \ \ uint64_t a0[2]; \ - memcpy_swap64le(a0, a, 2); \ - chunk3[0] = SWAP64LE(SWAP64LE(chunk2[0]) + a0[0]); \ - chunk3[1] = SWAP64LE(SWAP64LE(chunk2[1]) + a0[1]); \ + memcpy_swap64le(a0, a_, 2); \ + chunk3[0] = SWAP64LE(chunk2_old[0] + a0[0]); \ + chunk3[1] = SWAP64LE(chunk2_old[1] + a0[1]); \ \ uint64_t b0[2]; \ memcpy_swap64le(b0, b, 2); \ - chunk2[0] = SWAP64LE(SWAP64LE(chunk1_old[0]) + b0[0]); \ + chunk2[0] = SWAP64LE(chunk1_old[0] + b0[0]); \ chunk2[1] = SWAP64LE(SWAP64LE(chunk1_old[1]) + b0[1]); \ + if (variant >= 4) \ + { \ + uint64_t out_copy[2]; \ + memcpy_swap64le(out_copy, out, 2); \ + chunk1_old[0] ^= chunk2_old[0]; \ + chunk1_old[1] ^= chunk2_old[1]; \ + out_copy[0] ^= chunk3_old[0]; \ + out_copy[1] ^= chunk3_old[1]; \ + out_copy[0] ^= chunk1_old[0]; \ + out_copy[1] ^= chunk1_old[1]; \ + memcpy_swap64le(out, out_copy, 2); \ + } \ } while (0) #define VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr) \ @@ -172,7 +238,7 @@ extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *ex const uint64_t sqrt_input = SWAP64LE(((uint64_t*)(ptr))[0]) + division_result #define VARIANT2_INTEGER_MATH_SSE2(b, ptr) \ - do if (variant >= 2) \ + do if ((variant == 2) || (variant == 3)) \ { \ VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2(); \ @@ -182,7 +248,7 @@ extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *ex #if defined DBL_MANT_DIG && (DBL_MANT_DIG >= 50) // double precision floating point type has enough bits of precision on current platform #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ - do if (variant >= 2) \ + do if ((variant == 2) || (variant == 3)) \ { \ VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ VARIANT2_INTEGER_MATH_SQRT_STEP_FP64(); \ @@ -192,7 +258,7 @@ extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *ex // double precision floating point type is not good enough on current platform // fall back to the reference code (integer only) #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ - do if (variant >= 2) \ + do if ((variant == 2) || (variant == 3)) \ { \ VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ VARIANT2_INTEGER_MATH_SQRT_STEP_REF(); \ @@ -200,13 +266,13 @@ extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *ex #endif #define VARIANT2_2_PORTABLE() \ - if (variant >= 2) { \ + if (variant == 2 || variant == 3) { \ xor_blocks(long_state + (j ^ 0x10), d); \ xor_blocks(d, long_state + (j ^ 0x20)); \ } #define VARIANT2_2() \ - do if (variant >= 2) \ + do if (variant == 2 || variant == 3) \ { \ *U64(hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ *(U64(hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ @@ -214,6 +280,68 @@ extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *ex lo ^= SWAP64LE(*(U64(hp_state + (j ^ 0x20)) + 1)); \ } while (0) +#define V4_REG_LOAD(dst, src) \ + do { \ + memcpy((dst), (src), sizeof(v4_reg)); \ + if (sizeof(v4_reg) == sizeof(uint32_t)) \ + *(dst) = SWAP32LE(*(dst)); \ + else \ + *(dst) = SWAP64LE(*(dst)); \ + } while (0) + +#define VARIANT4_RANDOM_MATH_INIT() \ + v4_reg r[9]; \ + struct V4_Instruction code[NUM_INSTRUCTIONS_MAX + 1]; \ + int jit = use_v4_jit(); \ + do if (variant >= 4) \ + { \ + for (int i = 0; i < 4; ++i) \ + V4_REG_LOAD(r + i, (uint8_t*)(state.hs.w + 12) + sizeof(v4_reg) * i); \ + v4_random_math_init(code, height); \ + if (jit) \ + { \ + int ret = v4_generate_JIT_code(code, hp_jitfunc, 4096); \ + if (ret < 0) \ + local_abort("Error generating CryptonightR code"); \ + } \ + } while (0) + +#define VARIANT4_RANDOM_MATH(a, b, r, _b, _b1) \ + do if (variant >= 4) \ + { \ + uint64_t t[2]; \ + memcpy(t, b, sizeof(uint64_t)); \ + \ + if (sizeof(v4_reg) == sizeof(uint32_t)) \ + t[0] ^= SWAP64LE((r[0] + r[1]) | ((uint64_t)(r[2] + r[3]) << 32)); \ + else \ + t[0] ^= SWAP64LE((r[0] + r[1]) ^ (r[2] + r[3])); \ + \ + memcpy(b, t, sizeof(uint64_t)); \ + \ + V4_REG_LOAD(r + 4, a); \ + V4_REG_LOAD(r + 5, (uint64_t*)(a) + 1); \ + V4_REG_LOAD(r + 6, _b); \ + V4_REG_LOAD(r + 7, _b1); \ + V4_REG_LOAD(r + 8, (uint64_t*)(_b1) + 1); \ + \ + if (jit) \ + (*hp_jitfunc)(r); \ + else \ + v4_random_math(code, r); \ + \ + memcpy(t, a, sizeof(uint64_t) * 2); \ + \ + if (sizeof(v4_reg) == sizeof(uint32_t)) { \ + t[0] ^= SWAP64LE(r[2] | ((uint64_t)(r[3]) << 32)); \ + t[1] ^= SWAP64LE(r[0] | ((uint64_t)(r[1]) << 32)); \ + } else { \ + t[0] ^= SWAP64LE(r[2] ^ r[3]); \ + t[1] ^= SWAP64LE(r[0] ^ r[1]); \ + } \ + memcpy(a, t, sizeof(uint64_t) * 2); \ + } while (0) + #if !defined NO_AES && (defined(__x86_64__) || (defined(_MSC_VER) && defined(_WIN64))) // Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI @@ -298,6 +426,7 @@ extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *ex p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_INTEGER_MATH_SSE2(b, c); \ + VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ @@ -329,6 +458,9 @@ union cn_slow_hash_state THREADV uint8_t *hp_state = NULL; THREADV int hp_allocated = 0; +THREADV v4_random_math_JIT_func hp_jitfunc = NULL; +THREADV uint8_t *hp_jitfunc_memory = NULL; +THREADV int hp_jitfunc_allocated = 0; #if defined(_MSC_VER) #define cpuid(info,x) __cpuidex(info,x,0) @@ -638,6 +770,33 @@ void slow_hash_allocate_state(void) hp_allocated = 0; hp_state = (uint8_t *) malloc(MEMORY); } + + +#if defined(_MSC_VER) || defined(__MINGW32__) + hp_jitfunc_memory = (uint8_t *) VirtualAlloc(hp_jitfunc_memory, 4096 + 4095, + MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); +#else +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(__DragonFly__) || defined(__NetBSD__) + hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANON, 0, 0); +#else + hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); +#endif + if(hp_jitfunc_memory == MAP_FAILED) + hp_jitfunc_memory = NULL; +#endif + hp_jitfunc_allocated = 1; + if (hp_jitfunc_memory == NULL) + { + hp_jitfunc_allocated = 0; + hp_jitfunc_memory = malloc(4096 + 4095); + } + hp_jitfunc = (v4_random_math_JIT_func)((size_t)(hp_jitfunc_memory + 4095) & ~4095); +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + mprotect(hp_jitfunc, 4096, PROT_READ | PROT_WRITE | PROT_EXEC); +#endif } /** @@ -660,8 +819,22 @@ void slow_hash_free_state(void) #endif } + if(!hp_jitfunc_allocated) + free(hp_jitfunc_memory); + else + { +#if defined(_MSC_VER) || defined(__MINGW32__) + VirtualFree(hp_jitfunc_memory, 0, MEM_RELEASE); +#else + munmap(hp_jitfunc_memory, 4096 + 4095); +#endif + } + hp_state = NULL; hp_allocated = 0; + hp_jitfunc = NULL; + hp_jitfunc_memory = NULL; + hp_jitfunc_allocated = 0; } /** @@ -694,7 +867,7 @@ void slow_hash_free_state(void) * @param length the length in bytes of the data * @param hash a pointer to a buffer in which the final 256 bit hash will be stored */ -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { RDATA_ALIGN16 uint8_t expandedKey[240]; /* These buffers are aligned to use later with SSE functions */ @@ -730,6 +903,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int VARIANT1_INIT64(); VARIANT2_INIT64(); + VARIANT4_RANDOM_MATH_INIT(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -855,6 +1029,8 @@ void slow_hash_free_state(void) #define U64(x) ((uint64_t *) (x)) +#define hp_jitfunc ((v4_random_math_JIT_func)NULL) + STATIC INLINE void xor64(uint64_t *a, const uint64_t b) { *a ^= b; @@ -901,6 +1077,7 @@ union cn_slow_hash_state p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ + VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ @@ -1063,7 +1240,7 @@ STATIC INLINE void aligned_free(void *ptr) } #endif /* FORCE_USE_HEAP */ -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { RDATA_ALIGN16 uint8_t expandedKey[240]; @@ -1100,6 +1277,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int VARIANT1_INIT64(); VARIANT2_INIT64(); + VARIANT4_RANDOM_MATH_INIT(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -1278,10 +1456,11 @@ STATIC INLINE void xor_blocks(uint8_t* a, const uint8_t* b) U64(a)[1] ^= U64(b)[1]; } -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; + uint8_t a1[AES_BLOCK_SIZE]; uint8_t b[AES_BLOCK_SIZE * 2]; uint8_t c[AES_BLOCK_SIZE]; uint8_t c1[AES_BLOCK_SIZE]; @@ -1317,6 +1496,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int VARIANT1_INIT64(); VARIANT2_INIT64(); + VARIANT4_RANDOM_MATH_INIT(); // use aligned data memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len); @@ -1340,10 +1520,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int // Iteration 1 j = state_index(a); p = &long_state[j]; - aesb_single_round(p, p, a); - copy_block(c1, p); + aesb_single_round(p, c1, a); - VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); + copy_block(p, c1); xor_blocks(p, b); VARIANT1_1(p); @@ -1352,13 +1532,15 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int p = &long_state[j]; copy_block(c, p); + copy_block(a1, a); VARIANT2_PORTABLE_INTEGER_MATH(c, c1); + VARIANT4_RANDOM_MATH(a1, c, r, b, b + AES_BLOCK_SIZE); mul(c1, c, d); VARIANT2_2_PORTABLE(); - VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); - sum_half_blocks(a, d); - swap_blocks(a, c); - xor_blocks(a, c); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); + sum_half_blocks(a1, d); + swap_blocks(a1, c); + xor_blocks(a1, c); VARIANT1_2(U64(c) + 1); copy_block(p, c); @@ -1366,6 +1548,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int copy_block(b + AES_BLOCK_SIZE, b); } copy_block(b, c1); + copy_block(a, a1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1393,6 +1576,8 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #else // Portable implementation as a fallback +#define hp_jitfunc ((v4_random_math_JIT_func)NULL) + void slow_hash_allocate_state(void) { // Do nothing, this is just to maintain compatibility with the upgraded slow-hash.c @@ -1476,7 +1661,7 @@ union cn_slow_hash_state { }; #pragma pack(pop) -void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) { +void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) { #ifndef FORCE_USE_HEAP uint8_t long_state[MEMORY]; #else @@ -1486,6 +1671,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int union cn_slow_hash_state state; uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; + uint8_t a1[AES_BLOCK_SIZE]; uint8_t b[AES_BLOCK_SIZE * 2]; uint8_t c1[AES_BLOCK_SIZE]; uint8_t c2[AES_BLOCK_SIZE]; @@ -1505,6 +1691,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int VARIANT1_PORTABLE_INIT(); VARIANT2_PORTABLE_INIT(); + VARIANT4_RANDOM_MATH_INIT(); oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { @@ -1528,7 +1715,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int j = e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; copy_block(c1, &long_state[j]); aesb_single_round(c1, c1, a); - VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); copy_block(&long_state[j], c1); xor_blocks(&long_state[j], b); assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); @@ -1536,22 +1723,22 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int /* Iteration 2 */ j = e2i(c1, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; copy_block(c2, &long_state[j]); + copy_block(a1, a); VARIANT2_PORTABLE_INTEGER_MATH(c2, c1); + VARIANT4_RANDOM_MATH(a1, c2, r, b, b + AES_BLOCK_SIZE); mul(c1, c2, d); VARIANT2_2_PORTABLE(); - VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); - swap_blocks(a, c1); - sum_half_blocks(c1, d); - swap_blocks(c1, c2); - xor_blocks(c1, c2); + VARIANT2_PORTABLE_SHUFFLE_ADD(c1, a, long_state, j); + sum_half_blocks(a1, d); + swap_blocks(a1, c2); + xor_blocks(a1, c2); VARIANT1_2(c2 + 8); copy_block(&long_state[j], c2); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); if (variant >= 2) { copy_block(b + AES_BLOCK_SIZE, b); } - copy_block(b, a); - copy_block(a, c1); + copy_block(b, c1); + copy_block(a, a1); } memcpy(text, state.init, INIT_SIZE_BYTE); diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index b2dc3ffb2..7802fb67f 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,15 +34,6 @@ #include "hash-ops.h" -#ifdef _MSC_VER -#include <malloc.h> -#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) \ - && !defined(__NetBSD__) - #include <alloca.h> -#else - #include <stdlib.h> -#endif - /*** * Round to power of two, for count>=3 and for count being not too large (as reasonable for tree hash calculations) */ @@ -91,9 +82,8 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { size_t cnt = tree_hash_cnt( count ); - char (*ints)[HASH_SIZE]; - size_t ints_size = cnt * HASH_SIZE; - ints = alloca(ints_size); memset( ints , 0 , ints_size); // allocate, and zero out as extra protection for using uninitialized mem + char ints[cnt][HASH_SIZE]; + memset(ints, 0 , sizeof(ints)); // zero out as extra protection for using uninitialized mem memcpy(ints, hashes, (2 * cnt - count) * HASH_SIZE); diff --git a/src/crypto/variant4_random_math.h b/src/crypto/variant4_random_math.h new file mode 100644 index 000000000..f3e41a001 --- /dev/null +++ b/src/crypto/variant4_random_math.h @@ -0,0 +1,441 @@ +#ifndef VARIANT4_RANDOM_MATH_H +#define VARIANT4_RANDOM_MATH_H + +// Register size can be configured to either 32 bit (uint32_t) or 64 bit (uint64_t) +typedef uint32_t v4_reg; + +enum V4_Settings +{ + // Generate code with minimal theoretical latency = 45 cycles, which is equivalent to 15 multiplications + TOTAL_LATENCY = 15 * 3, + + // Always generate at least 60 instructions + NUM_INSTRUCTIONS_MIN = 60, + + // Never generate more than 70 instructions (final RET instruction doesn't count here) + NUM_INSTRUCTIONS_MAX = 70, + + // Available ALUs for MUL + // Modern CPUs typically have only 1 ALU which can do multiplications + ALU_COUNT_MUL = 1, + + // Total available ALUs + // Modern CPUs have 4 ALUs, but we use only 3 because random math executes together with other main loop code + ALU_COUNT = 3, +}; + +enum V4_InstructionList +{ + MUL, // a*b + ADD, // a+b + C, C is an unsigned 32-bit constant + SUB, // a-b + ROR, // rotate right "a" by "b & 31" bits + ROL, // rotate left "a" by "b & 31" bits + XOR, // a^b + RET, // finish execution + V4_INSTRUCTION_COUNT = RET, +}; + +// V4_InstructionDefinition is used to generate code from random data +// Every random sequence of bytes is a valid code +// +// There are 9 registers in total: +// - 4 variable registers +// - 5 constant registers initialized from loop variables +// This is why dst_index is 2 bits +enum V4_InstructionDefinition +{ + V4_OPCODE_BITS = 3, + V4_DST_INDEX_BITS = 2, + V4_SRC_INDEX_BITS = 3, +}; + +struct V4_Instruction +{ + uint8_t opcode; + uint8_t dst_index; + uint8_t src_index; + uint32_t C; +}; + +#ifndef FORCEINLINE +#if defined(__GNUC__) +#define FORCEINLINE __attribute__((always_inline)) inline +#elif defined(_MSC_VER) +#define FORCEINLINE __forceinline +#else +#define FORCEINLINE inline +#endif +#endif + +#ifndef UNREACHABLE_CODE +#if defined(__GNUC__) +#define UNREACHABLE_CODE __builtin_unreachable() +#elif defined(_MSC_VER) +#define UNREACHABLE_CODE __assume(false) +#else +#define UNREACHABLE_CODE +#endif +#endif + +// Random math interpreter's loop is fully unrolled and inlined to achieve 100% branch prediction on CPU: +// every switch-case will point to the same destination on every iteration of Cryptonight main loop +// +// This is about as fast as it can get without using low-level machine code generation +static FORCEINLINE void v4_random_math(const struct V4_Instruction* code, v4_reg* r) +{ + enum + { + REG_BITS = sizeof(v4_reg) * 8, + }; + +#define V4_EXEC(i) \ + { \ + const struct V4_Instruction* op = code + i; \ + const v4_reg src = r[op->src_index]; \ + v4_reg* dst = r + op->dst_index; \ + switch (op->opcode) \ + { \ + case MUL: \ + *dst *= src; \ + break; \ + case ADD: \ + *dst += src + op->C; \ + break; \ + case SUB: \ + *dst -= src; \ + break; \ + case ROR: \ + { \ + const uint32_t shift = src % REG_BITS; \ + *dst = (*dst >> shift) | (*dst << ((REG_BITS - shift) % REG_BITS)); \ + } \ + break; \ + case ROL: \ + { \ + const uint32_t shift = src % REG_BITS; \ + *dst = (*dst << shift) | (*dst >> ((REG_BITS - shift) % REG_BITS)); \ + } \ + break; \ + case XOR: \ + *dst ^= src; \ + break; \ + case RET: \ + return; \ + default: \ + UNREACHABLE_CODE; \ + break; \ + } \ + } + +#define V4_EXEC_10(j) \ + V4_EXEC(j + 0) \ + V4_EXEC(j + 1) \ + V4_EXEC(j + 2) \ + V4_EXEC(j + 3) \ + V4_EXEC(j + 4) \ + V4_EXEC(j + 5) \ + V4_EXEC(j + 6) \ + V4_EXEC(j + 7) \ + V4_EXEC(j + 8) \ + V4_EXEC(j + 9) + + // Generated program can have 60 + a few more (usually 2-3) instructions to achieve required latency + // I've checked all block heights < 10,000,000 and here is the distribution of program sizes: + // + // 60 27960 + // 61 105054 + // 62 2452759 + // 63 5115997 + // 64 1022269 + // 65 1109635 + // 66 153145 + // 67 8550 + // 68 4529 + // 69 102 + + // Unroll 70 instructions here + V4_EXEC_10(0); // instructions 0-9 + V4_EXEC_10(10); // instructions 10-19 + V4_EXEC_10(20); // instructions 20-29 + V4_EXEC_10(30); // instructions 30-39 + V4_EXEC_10(40); // instructions 40-49 + V4_EXEC_10(50); // instructions 50-59 + V4_EXEC_10(60); // instructions 60-69 + +#undef V4_EXEC_10 +#undef V4_EXEC +} + +// If we don't have enough data available, generate more +static FORCEINLINE void check_data(size_t* data_index, const size_t bytes_needed, int8_t* data, const size_t data_size) +{ + if (*data_index + bytes_needed > data_size) + { + hash_extra_blake(data, data_size, (char*) data); + *data_index = 0; + } +} + +// Generates as many random math operations as possible with given latency and ALU restrictions +// "code" array must have space for NUM_INSTRUCTIONS_MAX+1 instructions +static inline int v4_random_math_init(struct V4_Instruction* code, const uint64_t height) +{ + // MUL is 3 cycles, 3-way addition and rotations are 2 cycles, SUB/XOR are 1 cycle + // These latencies match real-life instruction latencies for Intel CPUs starting from Sandy Bridge and up to Skylake/Coffee lake + // + // AMD Ryzen has the same latencies except 1-cycle ROR/ROL, so it'll be a bit faster than Intel Sandy Bridge and newer processors + // Surprisingly, Intel Nehalem also has 1-cycle ROR/ROL, so it'll also be faster than Intel Sandy Bridge and newer processors + // AMD Bulldozer has 4 cycles latency for MUL (slower than Intel) and 1 cycle for ROR/ROL (faster than Intel), so average performance will be the same + // Source: https://www.agner.org/optimize/instruction_tables.pdf + const int op_latency[V4_INSTRUCTION_COUNT] = { 3, 2, 1, 2, 2, 1 }; + + // Instruction latencies for theoretical ASIC implementation + const int asic_op_latency[V4_INSTRUCTION_COUNT] = { 3, 1, 1, 1, 1, 1 }; + + // Available ALUs for each instruction + const int op_ALUs[V4_INSTRUCTION_COUNT] = { ALU_COUNT_MUL, ALU_COUNT, ALU_COUNT, ALU_COUNT, ALU_COUNT, ALU_COUNT }; + + int8_t data[32]; + memset(data, 0, sizeof(data)); + uint64_t tmp = SWAP64LE(height); + memcpy(data, &tmp, sizeof(uint64_t)); + data[20] = -38; // change seed + + // Set data_index past the last byte in data + // to trigger full data update with blake hash + // before we start using it + size_t data_index = sizeof(data); + + int code_size; + + // There is a small chance (1.8%) that register R8 won't be used in the generated program + // So we keep track of it and try again if it's not used + bool r8_used; + do { + int latency[9]; + int asic_latency[9]; + + // Tracks previous instruction and value of the source operand for registers R0-R3 throughout code execution + // byte 0: current value of the destination register + // byte 1: instruction opcode + // byte 2: current value of the source register + // + // Registers R4-R8 are constant and are treated as having the same value because when we do + // the same operation twice with two constant source registers, it can be optimized into a single operation + uint32_t inst_data[9] = { 0, 1, 2, 3, 0xFFFFFF, 0xFFFFFF, 0xFFFFFF, 0xFFFFFF, 0xFFFFFF }; + + bool alu_busy[TOTAL_LATENCY + 1][ALU_COUNT]; + bool is_rotation[V4_INSTRUCTION_COUNT]; + bool rotated[4]; + int rotate_count = 0; + + memset(latency, 0, sizeof(latency)); + memset(asic_latency, 0, sizeof(asic_latency)); + memset(alu_busy, 0, sizeof(alu_busy)); + memset(is_rotation, 0, sizeof(is_rotation)); + memset(rotated, 0, sizeof(rotated)); + is_rotation[ROR] = true; + is_rotation[ROL] = true; + + int num_retries = 0; + code_size = 0; + + int total_iterations = 0; + r8_used = false; + + // Generate random code to achieve minimal required latency for our abstract CPU + // Try to get this latency for all 4 registers + while (((latency[0] < TOTAL_LATENCY) || (latency[1] < TOTAL_LATENCY) || (latency[2] < TOTAL_LATENCY) || (latency[3] < TOTAL_LATENCY)) && (num_retries < 64)) + { + // Fail-safe to guarantee loop termination + ++total_iterations; + if (total_iterations > 256) + break; + + check_data(&data_index, 1, data, sizeof(data)); + + const uint8_t c = ((uint8_t*)data)[data_index++]; + + // MUL = opcodes 0-2 + // ADD = opcode 3 + // SUB = opcode 4 + // ROR/ROL = opcode 5, shift direction is selected randomly + // XOR = opcodes 6-7 + uint8_t opcode = c & ((1 << V4_OPCODE_BITS) - 1); + if (opcode == 5) + { + check_data(&data_index, 1, data, sizeof(data)); + opcode = (data[data_index++] >= 0) ? ROR : ROL; + } + else if (opcode >= 6) + { + opcode = XOR; + } + else + { + opcode = (opcode <= 2) ? MUL : (opcode - 2); + } + + uint8_t dst_index = (c >> V4_OPCODE_BITS) & ((1 << V4_DST_INDEX_BITS) - 1); + uint8_t src_index = (c >> (V4_OPCODE_BITS + V4_DST_INDEX_BITS)) & ((1 << V4_SRC_INDEX_BITS) - 1); + + const int a = dst_index; + int b = src_index; + + // Don't do ADD/SUB/XOR with the same register + if (((opcode == ADD) || (opcode == SUB) || (opcode == XOR)) && (a == b)) + { + // Use register R8 as source instead + b = 8; + src_index = 8; + } + + // Don't do rotation with the same destination twice because it's equal to a single rotation + if (is_rotation[opcode] && rotated[a]) + { + continue; + } + + // Don't do the same instruction (except MUL) with the same source value twice because all other cases can be optimized: + // 2xADD(a, b, C) = ADD(a, b*2, C1+C2), same for SUB and rotations + // 2xXOR(a, b) = NOP + if ((opcode != MUL) && ((inst_data[a] & 0xFFFF00) == (opcode << 8) + ((inst_data[b] & 255) << 16))) + { + continue; + } + + // Find which ALU is available (and when) for this instruction + int next_latency = (latency[a] > latency[b]) ? latency[a] : latency[b]; + int alu_index = -1; + while (next_latency < TOTAL_LATENCY) + { + for (int i = op_ALUs[opcode] - 1; i >= 0; --i) + { + if (!alu_busy[next_latency][i]) + { + // ADD is implemented as two 1-cycle instructions on a real CPU, so do an additional availability check + if ((opcode == ADD) && alu_busy[next_latency + 1][i]) + { + continue; + } + + // Rotation can only start when previous rotation is finished, so do an additional availability check + if (is_rotation[opcode] && (next_latency < rotate_count * op_latency[opcode])) + { + continue; + } + + alu_index = i; + break; + } + } + if (alu_index >= 0) + { + break; + } + ++next_latency; + } + + // Don't generate instructions that leave some register unchanged for more than 7 cycles + if (next_latency > latency[a] + 7) + { + continue; + } + + next_latency += op_latency[opcode]; + + if (next_latency <= TOTAL_LATENCY) + { + if (is_rotation[opcode]) + { + ++rotate_count; + } + + // Mark ALU as busy only for the first cycle when it starts executing the instruction because ALUs are fully pipelined + alu_busy[next_latency - op_latency[opcode]][alu_index] = true; + latency[a] = next_latency; + + // ASIC is supposed to have enough ALUs to run as many independent instructions per cycle as possible, so latency calculation for ASIC is simple + asic_latency[a] = ((asic_latency[a] > asic_latency[b]) ? asic_latency[a] : asic_latency[b]) + asic_op_latency[opcode]; + + rotated[a] = is_rotation[opcode]; + + inst_data[a] = code_size + (opcode << 8) + ((inst_data[b] & 255) << 16); + + code[code_size].opcode = opcode; + code[code_size].dst_index = dst_index; + code[code_size].src_index = src_index; + code[code_size].C = 0; + + if (src_index == 8) + { + r8_used = true; + } + + if (opcode == ADD) + { + // ADD instruction is implemented as two 1-cycle instructions on a real CPU, so mark ALU as busy for the next cycle too + alu_busy[next_latency - op_latency[opcode] + 1][alu_index] = true; + + // ADD instruction requires 4 more random bytes for 32-bit constant "C" in "a = a + b + C" + check_data(&data_index, sizeof(uint32_t), data, sizeof(data)); + uint32_t t; + memcpy(&t, data + data_index, sizeof(uint32_t)); + code[code_size].C = SWAP32LE(t); + data_index += sizeof(uint32_t); + } + + ++code_size; + if (code_size >= NUM_INSTRUCTIONS_MIN) + { + break; + } + } + else + { + ++num_retries; + } + } + + // ASIC has more execution resources and can extract as much parallelism from the code as possible + // We need to add a few more MUL and ROR instructions to achieve minimal required latency for ASIC + // Get this latency for at least 1 of the 4 registers + const int prev_code_size = code_size; + while ((code_size < NUM_INSTRUCTIONS_MAX) && (asic_latency[0] < TOTAL_LATENCY) && (asic_latency[1] < TOTAL_LATENCY) && (asic_latency[2] < TOTAL_LATENCY) && (asic_latency[3] < TOTAL_LATENCY)) + { + int min_idx = 0; + int max_idx = 0; + for (int i = 1; i < 4; ++i) + { + if (asic_latency[i] < asic_latency[min_idx]) min_idx = i; + if (asic_latency[i] > asic_latency[max_idx]) max_idx = i; + } + + const uint8_t pattern[3] = { ROR, MUL, MUL }; + const uint8_t opcode = pattern[(code_size - prev_code_size) % 3]; + latency[min_idx] = latency[max_idx] + op_latency[opcode]; + asic_latency[min_idx] = asic_latency[max_idx] + asic_op_latency[opcode]; + + code[code_size].opcode = opcode; + code[code_size].dst_index = min_idx; + code[code_size].src_index = max_idx; + code[code_size].C = 0; + ++code_size; + } + + // There is ~98.15% chance that loop condition is false, so this loop will execute only 1 iteration most of the time + // It never does more than 4 iterations for all block heights < 10,000,000 + } while (!r8_used || (code_size < NUM_INSTRUCTIONS_MIN) || (code_size > NUM_INSTRUCTIONS_MAX)); + + // It's guaranteed that NUM_INSTRUCTIONS_MIN <= code_size <= NUM_INSTRUCTIONS_MAX here + // Add final instruction to stop the interpreter + code[code_size].opcode = RET; + code[code_size].dst_index = 0; + code[code_size].src_index = 0; + code[code_size].C = 0; + + return code_size; +} + +#endif diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index 21445959d..5bb56e083 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index edbc2c561..a9aec163b 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index 021f84029..abf751b6e 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/account_boost_serialization.h b/src/cryptonote_basic/account_boost_serialization.h index 7379d787f..320a960dc 100644 --- a/src/cryptonote_basic/account_boost_serialization.h +++ b/src/cryptonote_basic/account_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/blobdatatype.h b/src/cryptonote_basic/blobdatatype.h index 7d6ff0187..20f6b2421 100644 --- a/src/cryptonote_basic/blobdatatype.h +++ b/src/cryptonote_basic/blobdatatype.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,7 +30,11 @@ #pragma once +#include <string> +#include "span.h" + namespace cryptonote { typedef std::string blobdata; + typedef epee::span<const char> blobdata_ref; } diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index eb73ab0ea..96398a90b 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -40,7 +40,8 @@ namespace cryptonote struct cryptonote_connection_context: public epee::net_utils::connection_context_base { cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0), - m_last_request_time(boost::posix_time::microsec_clock::universal_time()), m_callback_request_count(0), m_last_known_hash(crypto::null_hash) {} + m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0), + m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_rpc_port(0), m_anchor(false) {} enum state { @@ -59,6 +60,9 @@ namespace cryptonote boost::posix_time::ptime m_last_request_time; epee::copyable_atomic m_callback_request_count; //in debug purpose: problem with double callback rise crypto::hash m_last_known_hash; + uint32_t m_pruning_seed; + uint16_t m_rpc_port; + bool m_anchor; //size_t m_score; TODO: add score calculations }; @@ -81,4 +85,23 @@ namespace cryptonote } } + inline char get_protocol_state_char(cryptonote_connection_context::state s) + { + switch (s) + { + case cryptonote_connection_context::state_before_handshake: + return 'h'; + case cryptonote_connection_context::state_synchronizing: + return 's'; + case cryptonote_connection_context::state_standby: + return 'w'; + case cryptonote_connection_context::state_idle: + return 'i'; + case cryptonote_connection_context::state_normal: + return 'n'; + default: + return 'u'; + } + } + } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 196b20e2a..03caafbb0 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -152,6 +152,10 @@ namespace cryptonote }; + template<typename T> static inline unsigned int getpos(T &ar) { return 0; } + template<> inline unsigned int getpos(binary_archive<true> &ar) { return ar.stream().tellp(); } + template<> inline unsigned int getpos(binary_archive<false> &ar) { return ar.stream().tellg(); } + class transaction_prefix { @@ -201,9 +205,14 @@ namespace cryptonote mutable crypto::hash hash; mutable size_t blob_size; + bool pruned; + + std::atomic<unsigned int> unprunable_size; + std::atomic<unsigned int> prefix_size; + transaction(); - transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } - transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } return *this; } + transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } + transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; } virtual ~transaction(); void set_null(); void invalidate_hashes(); @@ -221,10 +230,18 @@ namespace cryptonote set_blob_size_valid(false); } + const unsigned int start_pos = getpos(ar); + FIELDS(*static_cast<transaction_prefix *>(this)) + if (std::is_same<Archive<W>, binary_archive<W>>()) + prefix_size = getpos(ar) - start_pos; + if (version == 1) { + if (std::is_same<Archive<W>, binary_archive<W>>()) + unprunable_size = getpos(ar) - start_pos; + ar.tag("signatures"); ar.begin_array(); PREPARE_CUSTOM_VECTOR_SERIALIZATION(vin.size(), signatures); @@ -232,7 +249,7 @@ namespace cryptonote if (!signatures_not_expected && vin.size() != signatures.size()) return false; - for (size_t i = 0; i < vin.size(); ++i) + if (!pruned) for (size_t i = 0; i < vin.size(); ++i) { size_t signature_size = get_signature_size(vin[i]); if (signatures_not_expected) @@ -263,7 +280,11 @@ namespace cryptonote bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); if (!r || !ar.stream().good()) return false; ar.end_object(); - if (rct_signatures.type != rct::RCTTypeNull) + + if (std::is_same<Archive<W>, binary_archive<W>>()) + unprunable_size = getpos(ar) - start_pos; + + if (!pruned && rct_signatures.type != rct::RCTTypeNull) { ar.tag("rctsig_prunable"); ar.begin_object(); @@ -274,6 +295,8 @@ namespace cryptonote } } } + if (!typename Archive<W>::is_saving()) + pruned = false; END_SERIALIZE() template<bool W, template <bool> class Archive> @@ -295,6 +318,8 @@ namespace cryptonote ar.end_object(); } } + if (!typename Archive<W>::is_saving()) + pruned = true; return true; } @@ -322,6 +347,9 @@ namespace cryptonote rct_signatures.type = rct::RCTTypeNull; set_hash_valid(false); set_blob_size_valid(false); + pruned = false; + unprunable_size = 0; + prefix_size = 0; } inline diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index 23a5bd5bd..e336cc1d1 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index 0b8131a7a..036273f0e 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 0725a2bb8..4ce9f3eeb 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -249,7 +249,6 @@ namespace boost { a & x.mask; a & x.amount; - // a & x.senderPk; // not serialized, as we do not use it in monero currently } template <class Archive> @@ -295,7 +294,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +322,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,7 +336,7 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeBulletproof) + if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2) a & x.p.pseudoOuts; } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 82428f196..f40464bd1 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -196,6 +196,7 @@ namespace cryptonote bool r = tx.serialize_base(ba); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, true), false, "Failed to expand transaction data"); + tx.invalidate_hashes(); return true; } //--------------------------------------------------------------- @@ -209,7 +210,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash) { std::stringstream ss; ss << tx_blob; @@ -221,10 +222,33 @@ namespace cryptonote //TODO: validate tx get_transaction_hash(tx, tx_hash); + return true; + } + //--------------------------------------------------------------- + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) + { + if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash)) + return false; get_transaction_prefix_hash(tx, tx_prefix_hash); return true; } //--------------------------------------------------------------- + bool is_v1_tx(const blobdata_ref& tx_blob) + { + uint64_t version; + const char* begin = static_cast<const char*>(tx_blob.data()); + const char* end = begin + tx_blob.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version <= 1; + } + //--------------------------------------------------------------- + bool is_v1_tx(const blobdata& tx_blob) + { + return is_v1_tx(blobdata_ref{tx_blob.data(), tx_blob.size()}); + } + //--------------------------------------------------------------- bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); @@ -372,6 +396,7 @@ namespace cryptonote for (const auto &bp: rv.p.bulletproofs) nlr += bp.L.size() * 2; const size_t bp_size = 32 * (9 + nlr); + CHECK_AND_ASSERT_THROW_MES_L1(n_outputs <= BULLETPROOF_MAX_OUTPUTS, "maximum number of outputs is " + std::to_string(BULLETPROOF_MAX_OUTPUTS) + " per transaction"); CHECK_AND_ASSERT_THROW_MES_L1(bp_base * n_padded_outputs >= bp_size, "Invalid bulletproof clawback"); const uint64_t bp_clawback = (bp_base * n_padded_outputs - bp_size) * 4 / 5; CHECK_AND_ASSERT_THROW_MES_L1(bp_clawback <= std::numeric_limits<uint64_t>::max() - blob_size, "Weight overflow"); @@ -861,6 +886,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + void get_blob_hash(const epee::span<const char>& blob, crypto::hash& res) + { + cn_fast_hash(blob.data(), blob.size(), res); + } + //--------------------------------------------------------------- void get_blob_hash(const blobdata& blob, crypto::hash& res) { cn_fast_hash(blob.data(), blob.size(), res); @@ -929,6 +959,13 @@ namespace cryptonote return h; } //--------------------------------------------------------------- + crypto::hash get_blob_hash(const epee::span<const char>& blob) + { + crypto::hash h = null_hash; + get_blob_hash(blob, h); + return h; + } + //--------------------------------------------------------------- crypto::hash get_transaction_hash(const transaction& t) { crypto::hash h = null_hash; @@ -941,26 +978,42 @@ namespace cryptonote return get_transaction_hash(t, res, NULL); } //--------------------------------------------------------------- - bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res) + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res) { if (t.version == 1) return false; - transaction &tt = const_cast<transaction&>(t); - std::stringstream ss; - binary_archive<true> ba(ss); - const size_t inputs = t.vin.size(); - const size_t outputs = t.vout.size(); - const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; - bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); - CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); - cryptonote::get_blob_hash(ss.str(), res); + static const crypto::hash empty_hash = { (char)0x70, (char)0xa4, (char)0x85, (char)0x5d, (char)0x04, (char)0xd8, (char)0xfa, (char)0x7b, (char)0x3b, (char)0x27, (char)0x82, (char)0xca, (char)0x53, (char)0xb6, (char)0x00, (char)0xe5, (char)0xc0, (char)0x03, (char)0xc7, (char)0xdc, (char)0xb2, (char)0x7d, (char)0x7e, (char)0x92, (char)0x3c, (char)0x23, (char)0xf7, (char)0x86, (char)0x01, (char)0x46, (char)0xd2, (char)0xc5 }; + const unsigned int unprunable_size = t.unprunable_size; + if (blob && unprunable_size) + { + CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes"); + if (blob->size() - unprunable_size == 0) + res = empty_hash; + else + cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); + } + else + { + transaction &tt = const_cast<transaction&>(t); + std::stringstream ss; + binary_archive<true> ba(ss); + const size_t inputs = t.vin.size(); + const size_t outputs = t.vout.size(); + const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; + bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); + CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); + if (ss.str().empty()) + res = empty_hash; + else + cryptonote::get_blob_hash(ss.str(), res); + } return true; } //--------------------------------------------------------------- - crypto::hash get_transaction_prunable_hash(const transaction& t) + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata) { crypto::hash res; - CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, res), "Failed to calculate tx prunable hash"); + CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash"); return res; } //--------------------------------------------------------------- @@ -1013,16 +1066,13 @@ namespace cryptonote transaction &tt = const_cast<transaction&>(t); + const blobdata blob = tx_to_blob(t); + const unsigned int unprunable_size = t.unprunable_size; + const unsigned int prefix_size = t.prefix_size; + // base rct - { - std::stringstream ss; - binary_archive<true> ba(ss); - const size_t inputs = t.vin.size(); - const size_t outputs = t.vout.size(); - bool r = tt.rct_signatures.serialize_rctsig_base(ba, inputs, outputs); - CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures base"); - cryptonote::get_blob_hash(ss.str(), hashes[1]); - } + CHECK_AND_ASSERT_MES(prefix_size <= unprunable_size && unprunable_size <= blob.size(), false, "Inconsistent transaction prefix, unprunable and blob sizes"); + cryptonote::get_blob_hash(epee::span<const char>(blob.data() + prefix_size, unprunable_size - prefix_size), hashes[1]); // prunable rct if (t.rct_signatures.type == rct::RCTTypeNull) @@ -1031,7 +1081,7 @@ namespace cryptonote } else { - CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, hashes[2]), false, "Failed to get tx prunable hash"); + CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, &blob, hashes[2]), false, "Failed to get tx prunable hash"); } // the tx hash is the hash of the 3 hashes @@ -1157,7 +1207,7 @@ namespace cryptonote } blobdata bd = get_block_hashing_blob(b); const int cn_variant = b.major_version >= 7 ? b.major_version - 6 : 0; - crypto::cn_slow_hash(bd.data(), bd.size(), res, cn_variant); + crypto::cn_slow_hash(bd.data(), bd.size(), res, cn_variant, height); return true; } //--------------------------------------------------------------- diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8d33b1ca4..40a9907be 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -51,8 +51,11 @@ namespace cryptonote crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); + bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx); + bool is_v1_tx(const blobdata_ref& tx_blob); + bool is_v1_tx(const blobdata& tx_blob); template<typename T> bool find_tx_extra_field_by_type(const std::vector<tx_extra_field>& tx_extra_fields, T& field, size_t index = 0) @@ -96,15 +99,17 @@ namespace cryptonote bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev); void get_blob_hash(const blobdata& blob, crypto::hash& res); + void get_blob_hash(const epee::span<const char>& blob, crypto::hash& res); crypto::hash get_blob_hash(const blobdata& blob); + crypto::hash get_blob_hash(const epee::span<const char>& blob); std::string short_hash_str(const crypto::hash& h); crypto::hash get_transaction_hash(const transaction& t); bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); - bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res); - crypto::hash get_transaction_prunable_hash(const transaction& t); + bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob, crypto::hash& res); + crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blob = NULL); bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash); diff --git a/src/cryptonote_basic/cryptonote_stat_info.h b/src/cryptonote_basic/cryptonote_stat_info.h index c0be2144e..4cc1bc764 100644 --- a/src/cryptonote_basic/cryptonote_stat_info.h +++ b/src/cryptonote_basic/cryptonote_stat_info.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,7 +34,7 @@ namespace cryptonote { - struct core_stat_info + struct core_stat_info_t { uint64_t tx_pool_size; uint64_t blockchain_height; @@ -50,4 +50,5 @@ namespace cryptonote KV_SERIALIZE(top_block_id_str) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<core_stat_info_t> core_stat_info; } diff --git a/src/cryptonote_basic/difficulty.cpp b/src/cryptonote_basic/difficulty.cpp index 55e3e93b3..89b748a83 100644 --- a/src/cryptonote_basic/difficulty.cpp +++ b/src/cryptonote_basic/difficulty.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/difficulty.h b/src/cryptonote_basic/difficulty.h index b06538467..8da355b22 100644 --- a/src/cryptonote_basic/difficulty.h +++ b/src/cryptonote_basic/difficulty.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index 87a394918..ebfe45f37 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -222,7 +222,6 @@ bool HardFork::reorganize_from_block_height(uint64_t height) if (height >= db.height()) return false; - db.set_batch_transactions(true); bool stop_batch = db.batch_start(); versions.clear(); @@ -306,11 +305,34 @@ bool HardFork::rescan_from_chain_height(uint64_t height) return rescan_from_block_height(height - 1); } +void HardFork::on_block_popped(uint64_t nblocks) +{ + CHECK_AND_ASSERT_THROW_MES(nblocks > 0, "nblocks must be greater than 0"); + + CRITICAL_REGION_LOCAL(lock); + + const uint64_t new_chain_height = db.height(); + const uint64_t old_chain_height = new_chain_height + nblocks; + uint8_t version; + uint64_t height; + for (height = old_chain_height - 1; height >= new_chain_height; --height) + { + versions.pop_back(); + version = db.get_hard_fork_version(height); + versions.push_front(version); + } + + // does not take voting into account + for (current_fork_index = heights.size() - 1; current_fork_index > 0; --current_fork_index) + if (height >= heights[current_fork_index].height) + break; +} + int HardFork::get_voted_fork_index(uint64_t height) const { CRITICAL_REGION_LOCAL(lock); uint32_t accumulated_votes = 0; - for (unsigned int n = heights.size() - 1; n > current_fork_index; --n) { + for (int n = heights.size() - 1; n >= 0; --n) { uint8_t v = heights[n].version; accumulated_votes += last_versions[v]; uint32_t threshold = (window_size * heights[n].threshold + 99) / 100; diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index a63a66976..123978b12 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -150,6 +150,16 @@ namespace cryptonote bool reorganize_from_chain_height(uint64_t height); /** + * @brief called when one or more blocks are popped from the blockchain + * + * The current fork will be updated by looking up the db, + * which is much cheaper than recomputing everything + * + * @param new_chain_height the height of the chain after popping + */ + void on_block_popped(uint64_t new_chain_height); + + /** * @brief returns current state at the given time * * Based on the approximate time of the last known hard fork, diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index f4de2ed7e..6628c8448 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -73,6 +73,9 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "miner" +#define AUTODETECT_WINDOW 10 // seconds +#define AUTODETECT_GAIN_THRESHOLD 1.02f // 2% + using namespace epee; #include "miner.h" @@ -108,6 +111,7 @@ namespace cryptonote m_starter_nonce(0), m_last_hr_merge_time(0), m_hashes(0), + m_total_hashes(0), m_do_print_hashrate(false), m_do_mining(false), m_current_hash_rate(0), @@ -115,7 +119,8 @@ namespace cryptonote m_min_idle_seconds(BACKGROUND_MINING_DEFAULT_MIN_IDLE_INTERVAL_IN_SECONDS), m_idle_threshold(BACKGROUND_MINING_DEFAULT_IDLE_THRESHOLD_PERCENTAGE), m_mining_target(BACKGROUND_MINING_DEFAULT_MINING_TARGET_PERCENTAGE), - m_miner_extra_sleep(BACKGROUND_MINING_DEFAULT_MINER_EXTRA_SLEEP_MILLIS) + m_miner_extra_sleep(BACKGROUND_MINING_DEFAULT_MINER_EXTRA_SLEEP_MILLIS), + m_block_reward(0) { } @@ -126,12 +131,13 @@ namespace cryptonote catch (...) { /* ignore */ } } //----------------------------------------------------------------------------------------------------- - bool miner::set_block_template(const block& bl, const difficulty_type& di, uint64_t height) + bool miner::set_block_template(const block& bl, const difficulty_type& di, uint64_t height, uint64_t block_reward) { CRITICAL_REGION_LOCAL(m_template_lock); m_template = bl; m_diffic = di; m_height = height; + m_block_reward = block_reward; ++m_template_no; m_starter_nonce = crypto::rand<uint32_t>(); return true; @@ -163,7 +169,7 @@ namespace cryptonote LOG_ERROR("Failed to get_block_template(), stopping mining"); return false; } - set_block_template(bl, di, height); + set_block_template(bl, di, height, expected_reward); return true; } //----------------------------------------------------------------------------------------------------- @@ -178,7 +184,12 @@ namespace cryptonote merge_hr(); return true; }); - + + m_autodetect_interval.do_call([&](){ + update_autodetection(); + return true; + }); + return true; } //----------------------------------------------------------------------------------------------------- @@ -209,6 +220,60 @@ namespace cryptonote m_hashes = 0; } //----------------------------------------------------------------------------------------------------- + void miner::update_autodetection() + { + if (m_threads_autodetect.empty()) + return; + + uint64_t now = epee::misc_utils::get_ns_count(); + uint64_t dt = now - m_threads_autodetect.back().first; + if (dt < AUTODETECT_WINDOW * 1000000000ull) + return; + + // work out how many more hashes we got + m_threads_autodetect.back().first = dt; + uint64_t dh = m_total_hashes - m_threads_autodetect.back().second; + m_threads_autodetect.back().second = dh; + float hs = dh / (dt / (float)1000000000); + MGINFO("Mining autodetection: " << m_threads_autodetect.size() << " threads: " << hs << " H/s"); + + // when we don't increase by at least 2%, stop, otherwise check next + // if N and N+1 have mostly the same hash rate, we want to "lighter" one + bool found = false; + if (m_threads_autodetect.size() > 1) + { + int previdx = m_threads_autodetect.size() - 2; + float previous_hs = m_threads_autodetect[previdx].second / (m_threads_autodetect[previdx].first / (float)1000000000); + if (previous_hs > 0 && hs / previous_hs < AUTODETECT_GAIN_THRESHOLD) + { + m_threads_total = m_threads_autodetect.size() - 1; + m_threads_autodetect.clear(); + MGINFO("Optimal number of threads seems to be " << m_threads_total); + found = true; + } + } + + if (!found) + { + // setup one more thread + m_threads_autodetect.push_back({now, m_total_hashes}); + m_threads_total = m_threads_autodetect.size(); + } + + // restart all threads + { + CRITICAL_REGION_LOCAL(m_threads_lock); + boost::interprocess::ipcdetail::atomic_write32(&m_stop, 1); + for(boost::thread& th: m_threads) + th.join(); + m_threads.clear(); + } + boost::interprocess::ipcdetail::atomic_write32(&m_stop, 0); + boost::interprocess::ipcdetail::atomic_write32(&m_thread_index, 0); + for(size_t i = 0; i != m_threads_total; i++) + m_threads.push_back(boost::thread(m_attrs, boost::bind(&miner::worker_thread, this))); + } + //----------------------------------------------------------------------------------------------------- void miner::init_options(boost::program_options::options_description& desc) { command_line::add_arg(desc, arg_extra_messages); @@ -242,7 +307,8 @@ namespace cryptonote } m_config_folder_path = boost::filesystem::path(command_line::get_arg(vm, arg_extra_messages)).parent_path().string(); m_config = AUTO_VAL_INIT(m_config); - epee::serialization::load_t_from_json_file(m_config, m_config_folder_path + "/" + MINER_CONFIG_FILE_NAME); + const std::string filename = m_config_folder_path + "/" + MINER_CONFIG_FILE_NAME; + CHECK_AND_ASSERT_MES(epee::serialization::load_t_from_json_file(m_config, filename), false, "Failed to load data from " << filename); MINFO("Loaded " << m_extra_messages.size() << " extra messages, current index " << m_config.current_extra_message_index); } @@ -295,8 +361,16 @@ namespace cryptonote //----------------------------------------------------------------------------------------------------- bool miner::start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background, bool ignore_battery) { + m_block_reward = 0; m_mine_address = adr; m_threads_total = static_cast<uint32_t>(threads_count); + if (threads_count == 0) + { + m_threads_autodetect.clear(); + m_threads_autodetect.push_back({epee::misc_utils::get_ns_count(), m_total_hashes}); + m_threads_total = 1; + } + m_attrs = attrs; m_starter_nonce = crypto::rand<uint32_t>(); CRITICAL_REGION_LOCAL(m_threads_lock); if(is_mining()) @@ -318,12 +392,15 @@ namespace cryptonote set_is_background_mining_enabled(do_background); set_ignore_battery(ignore_battery); - for(size_t i = 0; i != threads_count; i++) + for(size_t i = 0; i != m_threads_total; i++) { m_threads.push_back(boost::thread(attrs, boost::bind(&miner::worker_thread, this))); } - LOG_PRINT_L0("Mining has started with " << threads_count << " threads, good luck!" ); + if (threads_count == 0) + MINFO("Mining has started, autodetecting optimal number of threads, good luck!" ); + else + MINFO("Mining has started with " << threads_count << " threads, good luck!" ); if( get_is_background_mining_enabled() ) { @@ -360,7 +437,7 @@ namespace cryptonote if (!is_mining()) { - MDEBUG("Not mining - nothing to stop" ); + MTRACE("Not mining - nothing to stop" ); return true; } @@ -381,6 +458,7 @@ namespace cryptonote MINFO("Mining has been stopped, " << m_threads.size() << " finished" ); m_threads.clear(); + m_threads_autodetect.clear(); return true; } //----------------------------------------------------------------------------------------------------- @@ -506,6 +584,7 @@ namespace cryptonote } nonce+=m_threads_total; ++m_hashes; + ++m_total_hashes; } slow_hash_free_state(); MGINFO("Miner thread stopped ["<< th_local_index << "]"); diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index e16d9f3b8..08b1bd7f1 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -61,7 +61,7 @@ namespace cryptonote ~miner(); bool init(const boost::program_options::variables_map& vm, network_type nettype); static void init_options(boost::program_options::options_description& desc); - bool set_block_template(const block& bl, const difficulty_type& diffic, uint64_t height); + bool set_block_template(const block& bl, const difficulty_type& diffic, uint64_t height, uint64_t block_reward); bool on_block_chain_update(); bool start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background = false, bool ignore_battery = false); uint64_t get_speed() const; @@ -85,6 +85,7 @@ namespace cryptonote bool set_idle_threshold(uint8_t idle_threshold); uint8_t get_mining_target() const; bool set_mining_target(uint8_t mining_target); + uint64_t get_block_reward() const { return m_block_reward; } static constexpr uint8_t BACKGROUND_MINING_DEFAULT_IDLE_THRESHOLD_PERCENTAGE = 90; static constexpr uint8_t BACKGROUND_MINING_MIN_IDLE_THRESHOLD_PERCENTAGE = 50; @@ -103,6 +104,7 @@ namespace cryptonote bool worker_thread(); bool request_block_template(); void merge_hr(); + void update_autodetection(); struct miner_config { @@ -132,16 +134,20 @@ namespace cryptonote account_public_address m_mine_address; epee::math_helper::once_a_time_seconds<5> m_update_block_template_interval; epee::math_helper::once_a_time_seconds<2> m_update_merge_hr_interval; + epee::math_helper::once_a_time_seconds<1> m_autodetect_interval; std::vector<blobdata> m_extra_messages; miner_config m_config; std::string m_config_folder_path; std::atomic<uint64_t> m_last_hr_merge_time; std::atomic<uint64_t> m_hashes; + std::atomic<uint64_t> m_total_hashes; std::atomic<uint64_t> m_current_hash_rate; epee::critical_section m_last_hash_rates_lock; std::list<uint64_t> m_last_hash_rates; bool m_do_print_hashrate; bool m_do_mining; + std::vector<std::pair<uint64_t, uint64_t>> m_threads_autodetect; + boost::thread::attributes m_attrs; // background mining stuffs .. @@ -164,5 +170,6 @@ namespace cryptonote static bool get_process_time(uint64_t& total_time); static uint8_t get_percent_of_total(uint64_t some_time, uint64_t total_time); static boost::logic::tribool on_battery_power(); + std::atomic<uint64_t> m_block_reward; }; } diff --git a/src/cryptonote_basic/subaddress_index.h b/src/cryptonote_basic/subaddress_index.h index 9b71448f9..99933e229 100644 --- a/src/cryptonote_basic/subaddress_index.h +++ b/src/cryptonote_basic/subaddress_index.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/tx_extra.h b/src/cryptonote_basic/tx_extra.h index 009e35ebe..ecb4c6040 100644 --- a/src/cryptonote_basic/tx_extra.h +++ b/src/cryptonote_basic/tx_extra.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_basic/verification_context.h b/src/cryptonote_basic/verification_context.h index 8d2b633a2..36b63f254 100644 --- a/src/cryptonote_basic/verification_context.h +++ b/src/cryptonote_basic/verification_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 496678b5e..b6087de22 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -59,6 +59,8 @@ #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 60000 //size of block (bytes) after which reward for block calculated using block size #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 20000 //size of block (bytes) after which reward for block calculated using block size - before first fork #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 300000 //size of block (bytes) after which reward for block calculated using block size - second change, from v5 +#define CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE 100000 // size in blocks of the long term block weight median window +#define CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR 50 #define CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE 600 #define CRYPTONOTE_DISPLAY_DECIMAL_POINT 12 // COIN - number of smallest units in one coin @@ -108,11 +110,13 @@ #define P2P_DEFAULT_PACKET_MAX_SIZE 50000000 //50000000 bytes maximum packet size #define P2P_DEFAULT_PEERS_IN_HANDSHAKE 250 #define P2P_DEFAULT_CONNECTION_TIMEOUT 5000 //5 seconds +#define P2P_DEFAULT_SOCKS_CONNECT_TIMEOUT 45 // seconds #define P2P_DEFAULT_PING_CONNECTION_TIMEOUT 2000 //2 seconds #define P2P_DEFAULT_INVOKE_TIMEOUT 60*2*1000 //2 minutes #define P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT 5000 //5 seconds #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT 70 #define P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT 2 +#define P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT 2 #define P2P_DEFAULT_LIMIT_RATE_UP 2048 // kB/s #define P2P_DEFAULT_LIMIT_RATE_DOWN 8192 // kB/s @@ -141,6 +145,8 @@ #define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 #define HF_VERSION_PER_BYTE_FEE 8 +#define HF_VERSION_SMALLER_BP 10 +#define HF_VERSION_LONG_TERM_BLOCK_WEIGHT 10 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 @@ -150,6 +156,11 @@ #define BULLETPROOF_MAX_OUTPUTS 16 +#define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase +#define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved +#define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved +//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + // New constants are intended to go here namespace config { diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 231489fdb..fb96de226 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index bbac20eaa..3e903cd12 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -53,6 +53,8 @@ #include "ringct/rctSigs.h" #include "common/perf_timer.h" #include "common/notify.h" +#include "common/varint.h" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -113,6 +115,12 @@ static const struct { // version 9 starts from block 1686275, which is on or around the 19th of October, 2018. Fork time finalised on 2018-09-02. { 9, 1686275, 0, 1535889548 }, + + // version 10 starts from block 1788000, which is on or around the 9th of March, 2019. Fork time finalised on 2019-02-10. + { 10, 1788000, 0, 1549792439 }, + + // version 11 starts from block 1788720, which is on or around the 10th of March, 2019. Fork time finalised on 2019-02-15. + { 11, 1788720, 0, 1550225678 }, }; static const uint64_t mainnet_hard_fork_version_1_till = 1009826; @@ -137,6 +145,8 @@ static const struct { { 7, 1057027, 0, 1512211236 }, { 8, 1057058, 0, 1533211200 }, { 9, 1057778, 0, 1533297600 }, + { 10, 1154318, 0, 1550153694 }, + { 11, 1155038, 0, 1550225678 }, }; static const uint64_t testnet_hard_fork_version_1_till = 624633; @@ -158,12 +168,16 @@ static const struct { { 7, 37000, 0, 1521600000 }, { 8, 176456, 0, 1537821770 }, { 9, 177176, 0, 1537821771 }, + { 10, 269000, 0, 1550153694 }, + { 11, 269720, 0, 1550225678 }, }; //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), + m_long_term_block_weights_window(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), + m_long_term_effective_median_block_weight(0), m_difficulty_for_next_block_top_hash(crypto::null_hash), m_difficulty_for_next_block(1), m_btc_valid(false) @@ -173,7 +187,8 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : //------------------------------------------------------------------ Blockchain::~Blockchain() { - deinit(); + try { deinit(); } + catch (const std::exception &e) { /* ignore */ } } //------------------------------------------------------------------ bool Blockchain::have_tx(const crypto::hash &id) const @@ -432,9 +447,9 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline uint64_t top_block_timestamp = m_db->get_top_block_timestamp(); uint64_t timestamp_diff = time(NULL) - top_block_timestamp; - // genesis block has no timestamp, could probably change it to have timestamp of 1341378000... + // genesis block has no timestamp, could probably change it to have timestamp of 1397818133... if(!top_block_timestamp) - timestamp_diff = time(NULL) - 1341378000; + timestamp_diff = time(NULL) - 1397818133; // create general purpose async service queue @@ -453,8 +468,8 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline uint64_t num_popped_blocks = 0; while (!m_db->is_read_only()) { - const uint64_t top_height = m_db->height() - 1; - const crypto::hash top_id = m_db->top_block_hash(); + uint64_t top_height; + const crypto::hash top_id = m_db->top_block_hash(&top_height); const block top_block = m_db->get_top_block(); const uint8_t ideal_hf_version = get_ideal_hard_fork_version(top_height); if (ideal_hf_version <= 1 || ideal_hf_version == top_block.major_version) @@ -494,10 +509,16 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline { m_timestamps_and_difficulties_height = 0; m_hardfork->reorganize_from_chain_height(get_current_blockchain_height()); - m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); + uint64_t top_block_height; + crypto::hash top_block_hash = get_tail_id(top_block_height); + m_tx_pool.on_blockchain_dec(top_block_height, top_block_hash); } - update_next_cumulative_weight_limit(); + if (test_options && test_options->long_term_block_weight_window) + m_long_term_block_weights_window = test_options->long_term_block_weight_window; + + if (!update_next_cumulative_weight_limit()) + return false; return true; } //------------------------------------------------------------------ @@ -641,9 +662,18 @@ block Blockchain::pop_block_from_blockchain() throw; } + // make sure the hard fork object updates its current version + m_hardfork->on_block_popped(1); + // return transactions from popped block to the tx_pool + size_t pruned = 0; for (transaction& tx : popped_txs) { + if (tx.pruned) + { + ++pruned; + continue; + } if (!is_coinbase(tx)) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -651,12 +681,7 @@ block Blockchain::pop_block_from_blockchain() // FIXME: HardFork // Besides the below, popping a block should also remove the last entry // in hf_versions. - // - // FIXME: HardFork - // This is not quite correct, as we really want to add the txes - // to the pool based on the version determined after all blocks - // are popped. - uint8_t version = get_current_hard_fork_version(); + uint8_t version = get_ideal_hard_fork_version(m_db->height()); // We assume that if they were in a block, the transactions are already // known to the network as a whole. However, if we had mined that block, @@ -670,14 +695,18 @@ block Blockchain::pop_block_from_blockchain() } } } + if (pruned) + MWARNING(pruned << " pruned txes could not be added back to the txpool"); m_blocks_longhash_table.clear(); m_scan_table.clear(); m_blocks_txs_check.clear(); m_check_txin_table.clear(); - update_next_cumulative_weight_limit(); - m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); + uint64_t top_block_height; + crypto::hash top_block_hash = get_tail_id(top_block_height); + m_tx_pool.on_blockchain_dec(top_block_height, top_block_hash); invalidate_block_template_cache(); return popped_block; @@ -695,7 +724,8 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) block_verification_context bvc = boost::value_initialized<block_verification_context>(); add_new_block(b, bvc); - update_next_cumulative_weight_limit(); + if (!update_next_cumulative_weight_limit()) + return false; return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ @@ -703,8 +733,7 @@ crypto::hash Blockchain::get_tail_id(uint64_t& height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - height = m_db->height() - 1; - return get_tail_id(); + return m_db->top_block_hash(&height); } //------------------------------------------------------------------ crypto::hash Blockchain::get_tail_id() const @@ -865,8 +894,9 @@ difficulty_type Blockchain::get_difficulty_for_next_block() CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector<uint64_t> timestamps; std::vector<difficulty_type> difficulties; - auto height = m_db->height(); - top_hash = get_tail_id(); // get it again now that we have the lock + uint64_t height; + top_hash = get_tail_id(height); // get it again now that we have the lock + ++height; // top block height to blockchain height // ND: Speedup // 1. Keep a list of the last 735 (or less) blocks that is used to compute difficulty, // then when the next block difficulty is queried, push the latest height data and @@ -921,8 +951,10 @@ difficulty_type Blockchain::get_difficulty_for_next_block() //------------------------------------------------------------------ std::vector<time_t> Blockchain::get_last_block_timestamps(unsigned int blocks) const { - std::vector<time_t> timestamps(blocks); uint64_t height = m_db->height(); + if (blocks > height) + blocks = height; + std::vector<time_t> timestamps(blocks); while (blocks--) timestamps[blocks] = m_db->get_block_timestamp(height - blocks - 1); return timestamps; @@ -1038,6 +1070,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash:: } // if we're to keep the disconnected blocks, add them as alternates + const size_t discarded_blocks = disconnected_chain.size(); if(!discard_disconnected_chain) { //pushing old chain as alternative chain @@ -1062,6 +1095,11 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash:: m_hardfork->reorganize_from_chain_height(split_height); + std::shared_ptr<tools::Notify> reorg_notify = m_reorg_notify; + if (reorg_notify) + reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), + "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height()); return true; } @@ -1186,7 +1224,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } } - std::vector<size_t> last_blocks_weights; + std::vector<uint64_t> last_blocks_weights; get_last_n_blocks_weights(last_blocks_weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); if (!get_block_reward(epee::misc_utils::median(last_blocks_weights), cumulative_block_weight, already_generated_coins, base_reward, version)) { @@ -1199,7 +1237,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return false; } // From hard fork 2, we allow a miner to claim less block reward than is allowed, in case a miner wants less dust - if (m_hardfork->get_current_version() < 2) + if (version < 2) { if(base_reward + fee != money_in_use) { @@ -1221,7 +1259,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } //------------------------------------------------------------------ // get the block weights of the last <count> blocks, and return by reference <sz>. -void Blockchain::get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const +void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1712,7 +1750,6 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO for (auto& bl: blocks) { std::vector<crypto::hash> missed_tx_ids; - std::vector<cryptonote::blobdata> txs; rsp.blocks.push_back(block_complete_entry()); block_complete_entry& e = rsp.blocks.back(); @@ -1740,7 +1777,6 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO e.block = std::move(bl.first); } //get and pack other transactions, if needed - std::vector<cryptonote::blobdata> txs; get_transactions_blobs(arg.txs, rsp.txs, rsp.missed_ids); m_db->block_txn_stop(); @@ -1774,11 +1810,12 @@ uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const uint64_t num_outs = m_db->get_num_outputs(amount); // ensure we don't include outputs that aren't yet eligible to be used // outpouts are sorted by height + const uint64_t blockchain_height = m_db->height(); while (num_outs > 0) { const tx_out_index toi = m_db->get_output_tx_and_index(amount, num_outs - 1); const uint64_t height = m_db->get_tx_block_height(toi.first); - if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height()) + if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= blockchain_height) break; --num_outs; } @@ -2043,6 +2080,51 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con return true; } //------------------------------------------------------------------ +size_t get_transaction_version(const cryptonote::blobdata &bd) +{ + size_t version; + const char* begin = static_cast<const char*>(bd.data()); + const char* end = begin + bd.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version; +} +//------------------------------------------------------------------ +template<class t_ids_container, class t_tx_container, class t_missed_container> +bool Blockchain::get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + reserve_container(txs, txs_ids.size()); + for (const auto& tx_hash : txs_ids) + { + try + { + cryptonote::blobdata tx; + if (m_db->get_pruned_tx_blob(tx_hash, tx)) + { + txs.push_back(std::make_tuple(tx_hash, std::move(tx), crypto::null_hash, cryptonote::blobdata())); + if (!is_v1_tx(std::get<1>(txs.back())) && !m_db->get_prunable_tx_hash(tx_hash, std::get<2>(txs.back()))) + { + MERROR("Prunable data hash not found for " << tx_hash); + return false; + } + if (!m_db->get_prunable_tx_blob(tx_hash, std::get<3>(txs.back()))) + std::get<3>(txs.back()).clear(); + } + else + missed_txs.push_back(tx_hash); + } + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ template<class t_ids_container, class t_tx_container, class t_missed_container> bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { @@ -2091,9 +2173,12 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc m_db->block_txn_start(true); current_height = get_current_blockchain_height(); + const uint32_t pruning_seed = get_blockchain_pruning_seed(); + start_height = tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed); + uint64_t stop_height = tools::get_next_pruned_block_height(start_height, current_height, pruning_seed); size_t count = 0; - hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); - for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + hashes.reserve(std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); + for(size_t i = start_height; i < stop_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); } @@ -2298,7 +2383,7 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai return true; } //------------------------------------------------------------------ -bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2308,16 +2393,25 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); return false; } + indexs = m_db->get_tx_amount_output_indices(tx_index, n_txes); + CHECK_AND_ASSERT_MES(n_txes == indexs.size(), false, "Wrong indexs size"); - // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts - indexs = m_db->get_tx_amount_output_indices(tx_index); - if (indexs.empty()) + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + uint64_t tx_index; + if (!m_db->tx_exists(tx_id, tx_index)) { - // empty indexs is only valid if the vout is empty, which is legal but rare - cryptonote::transaction tx = m_db->get_tx(tx_id); - CHECK_AND_ASSERT_MES(tx.vout.empty(), false, "internal error: global indexes for transaction " << tx_id << " is empty, and tx vout is not"); + MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + return false; } - + std::vector<std::vector<uint64_t>> indices = m_db->get_tx_amount_output_indices(tx_index, 1); + CHECK_AND_ASSERT_MES(indices.size() == 1, false, "Wrong indices size"); + indexs = indices.front(); return true; } //------------------------------------------------------------------ @@ -2449,6 +2543,30 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context } } + // from v10, allow bulletproofs v2 + if (hf_version < HF_VERSION_SMALLER_BP) { + if (tx.version >= 2) { + if (tx.rct_signatures.type == rct::RCTTypeBulletproof2) + { + MERROR_VER("Ringct type " << (unsigned)rct::RCTTypeBulletproof2 << " is not allowed before v" << HF_VERSION_SMALLER_BP); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v11, allow only bulletproofs v2 + if (hf_version > HF_VERSION_SMALLER_BP) { + if (tx.version >= 2) { + if (tx.rct_signatures.type == rct::RCTTypeBulletproof) + { + MERROR_VER("Ringct type " << (unsigned)rct::RCTTypeBulletproof << " is not allowed from v" << (HF_VERSION_SMALLER_BP + 1)); + tvc.m_invalid_output = true; + return false; + } + } + } + return true; } //------------------------------------------------------------------ @@ -2489,7 +2607,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2515,7 +2633,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2789,6 +2907,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } case rct::RCTTypeSimple: case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: { // check all this, either reconstructed (so should really pass), or not { @@ -2997,6 +3116,7 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); + const uint64_t blockchain_height = m_db->height(); uint64_t median = 0; uint64_t already_generated_coins = 0; @@ -3004,7 +3124,7 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const if (version >= HF_VERSION_DYNAMIC_FEE) { median = m_current_block_cumul_weight_limit / 2; - already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + already_generated_coins = blockchain_height ? m_db->get_block_already_generated_coins(blockchain_height - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; } @@ -3012,7 +3132,8 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const uint64_t needed_fee; if (version >= HF_VERSION_PER_BYTE_FEE) { - uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, median, version); + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_effective_median_block_weight : median, version); MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); needed_fee = tx_weight * fee_per_byte; // quantize fee up to 8 decimals @@ -3049,6 +3170,7 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const { const uint8_t version = get_current_hard_fork_version(); + const uint64_t db_height = m_db->height(); if (version < HF_VERSION_DYNAMIC_FEE) return FEE_PER_KB; @@ -3057,7 +3179,7 @@ uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1; const uint64_t min_block_weight = get_min_block_weight(version); - std::vector<size_t> weights; + std::vector<uint64_t> weights; get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); weights.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) @@ -3067,7 +3189,7 @@ uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const if(median <= min_block_weight) median = min_block_weight; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + uint64_t already_generated_coins = db_height ? m_db->get_block_already_generated_coins(db_height - 1) : 0; uint64_t base_reward; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) { @@ -3075,7 +3197,8 @@ uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_base_fee(base_reward, median, version); + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_effective_median_block_weight : median, version); const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; @@ -3207,14 +3330,15 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons return false; } + const auto h = m_db->height(); + // if not enough blocks, no proper median yet, return true - if(m_db->height() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) + if(h < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) { return true; } std::vector<uint64_t> timestamps; - auto h = m_db->height(); // need most recent 60 blocks, get index of first of those size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; @@ -3227,7 +3351,7 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons return check_block_timestamp(timestamps, b, median_ts); } //------------------------------------------------------------------ -void Blockchain::return_tx_to_pool(std::vector<transaction> &txs) +void Blockchain::return_tx_to_pool(std::vector<std::pair<transaction, blobdata>> &txs) { uint8_t version = get_current_hard_fork_version(); for (auto& tx : txs) @@ -3238,9 +3362,11 @@ void Blockchain::return_tx_to_pool(std::vector<transaction> &txs) // that might not be always true. Unlikely though, and always relaying // these again might cause a spike of traffic as many nodes re-relay // all the transactions in a popped block when a reorg happens. - if (!m_tx_pool.add_tx(tx, tvc, true, true, false, version)) + const size_t weight = get_transaction_weight(tx.first, tx.second.size()); + const crypto::hash tx_hash = get_transaction_hash(tx.first); + if (!m_tx_pool.add_tx(tx.first, tx_hash, tx.second, weight, tvc, true, true, false, version)) { - MERROR("Failed to return taken transaction with hash: " << get_transaction_hash(tx) << " to tx_pool"); + MERROR("Failed to return taken transaction with hash: " << get_transaction_hash(tx.first) << " to tx_pool"); } } } @@ -3253,11 +3379,12 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) for (const auto &txid: txids) { cryptonote::transaction tx; + cryptonote::blobdata txblob; size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) + if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -3280,9 +3407,12 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& static bool seen_future_version = false; m_db->block_txn_start(true); - if(bl.prev_id != get_tail_id()) + uint64_t blockchain_height; + const crypto::hash top_hash = get_tail_id(blockchain_height); + ++blockchain_height; // block height to chain height + if(bl.prev_id != top_hash) { - MERROR_VER("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << get_tail_id()); + MERROR_VER("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << top_hash); bvc.m_verifivation_failed = true; leave: m_db->block_txn_stop(); @@ -3351,15 +3481,14 @@ leave: bool precomputed = false; bool fast_check = false; #if defined(PER_BLOCK_CHECKPOINT) - if (m_db->height() < m_blocks_hash_check.size()) + if (blockchain_height < m_blocks_hash_check.size()) { - auto hash = get_block_hash(bl); - const auto &expected_hash = m_blocks_hash_check[m_db->height()]; + const auto &expected_hash = m_blocks_hash_check[blockchain_height]; if (expected_hash != crypto::null_hash) { - if (memcmp(&hash, &expected_hash, sizeof(hash)) != 0) + if (memcmp(&id, &expected_hash, sizeof(hash)) != 0) { - MERROR_VER("Block with id is INVALID: " << id); + MERROR_VER("Block with id is INVALID: " << id << ", expected " << expected_hash); bvc.m_verifivation_failed = true; goto leave; } @@ -3367,7 +3496,7 @@ leave: } else { - MCINFO("verify", "No pre-validated hash at height " << m_db->height() << ", verifying fully"); + MCINFO("verify", "No pre-validated hash at height " << blockchain_height << ", verifying fully"); } } else @@ -3380,12 +3509,12 @@ leave: proof_of_work = it->second; } else - proof_of_work = get_block_longhash(bl, m_db->height()); + proof_of_work = get_block_longhash(bl, blockchain_height); // validate proof_of_work versus difficulty target if(!check_hash(proof_of_work, current_diffic)) { - MERROR_VER("Block with id: " << id << std::endl << "does not have enough proof of work: " << proof_of_work << std::endl << "unexpected difficulty: " << current_diffic); + MERROR_VER("Block with id: " << id << std::endl << "does not have enough proof of work: " << proof_of_work << " at height " << blockchain_height << ", unexpected difficulty: " << current_diffic); bvc.m_verifivation_failed = true; goto leave; } @@ -3393,9 +3522,9 @@ leave: // If we're at a checkpoint, ensure that our hardcoded checkpoint hash // is correct. - if(m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height())) + if(m_checkpoints.is_in_checkpoint_zone(blockchain_height)) { - if(!m_checkpoints.check_block(get_current_blockchain_height(), id)) + if(!m_checkpoints.check_block(blockchain_height, id)) { LOG_ERROR("CHECKPOINT VALIDATION FAILED"); bvc.m_verifivation_failed = true; @@ -3410,7 +3539,7 @@ leave: TIME_MEASURE_START(t3); // sanity check basic miner tx properties; - if(!prevalidate_miner_transaction(bl, m_db->height())) + if(!prevalidate_miner_transaction(bl, blockchain_height)) { MERROR_VER("Block with id: " << id << " failed to pass prevalidation"); bvc.m_verifivation_failed = true; @@ -3420,7 +3549,7 @@ leave: size_t coinbase_weight = get_transaction_weight(bl.miner_tx); size_t cumulative_block_weight = coinbase_weight; - std::vector<transaction> txs; + std::vector<std::pair<transaction, blobdata>> txs; key_images_container keys; uint64_t fee_summary = 0; @@ -3439,7 +3568,8 @@ leave: txs.reserve(bl.tx_hashes.size()); for (const crypto::hash& tx_id : bl.tx_hashes) { - transaction tx; + transaction tx_tmp; + blobdata txblob; size_t tx_weight = 0; uint64_t fee = 0; bool relayed = false, do_not_relay = false, double_spend_seen = false; @@ -3459,7 +3589,7 @@ leave: TIME_MEASURE_START(bb); // get transaction with hash <tx_id> from tx_pool - if(!m_tx_pool.take_tx(tx_id, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) + if(!m_tx_pool.take_tx(tx_id, tx_tmp, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; @@ -3472,7 +3602,8 @@ leave: // add the transaction to the temp list of transactions, so we can either // store the list of transactions all at once or return the ones we've // taken from the tx_pool back to it if the block fails verification. - txs.push_back(tx); + txs.push_back(std::make_pair(std::move(tx_tmp), std::move(txblob))); + transaction &tx = txs.back().first; TIME_MEASURE_START(dd); // FIXME: the storage should not be responsible for validation. @@ -3537,7 +3668,7 @@ leave: TIME_MEASURE_START(vmt); uint64_t base_reward = 0; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + uint64_t already_generated_coins = blockchain_height ? m_db->get_block_already_generated_coins(blockchain_height - 1) : 0; if(!validate_miner_transaction(bl, cumulative_block_weight, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) { MERROR_VER("Block with id: " << id << " has incorrect miner transaction"); @@ -3558,8 +3689,8 @@ leave: // at MONEY_SUPPLY. already_generated_coins is only used to compute the block subsidy and MONEY_SUPPLY yields a // subsidy of 0 under the base formula and therefore the minimum subsidy >0 in the tail state. already_generated_coins = base_reward < (MONEY_SUPPLY-already_generated_coins) ? already_generated_coins + base_reward : MONEY_SUPPLY; - if(m_db->height()) - cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height() - 1); + if(blockchain_height) + cumulative_difficulty += m_db->get_block_cumulative_difficulty(blockchain_height - 1); TIME_MEASURE_FINISH(block_processing_time); if(precomputed) @@ -3572,7 +3703,9 @@ leave: { try { - new_height = m_db->add_block(bl, block_weight, cumulative_difficulty, already_generated_coins, txs); + uint64_t long_term_block_weight = get_next_long_term_block_weight(block_weight); + cryptonote::blobdata bd = cryptonote::block_to_blob(bl); + new_height = m_db->add_block(std::make_pair(std::move(bl), std::move(bd)), block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { @@ -3598,7 +3731,12 @@ leave: TIME_MEASURE_FINISH(addblock); // do this after updating the hard fork state since the weight limit may change due to fork - update_next_cumulative_weight_limit(); + if (!update_next_cumulative_weight_limit()) + { + MERROR("Failed to update next cumulative weight limit"); + pop_block_from_blockchain(); + return false; + } MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); if(m_show_time_stats) @@ -3620,33 +3758,140 @@ leave: std::shared_ptr<tools::Notify> block_notify = m_block_notify; if (block_notify) - block_notify->notify(epee::string_tools::pod_to_hex(id).c_str()); + block_notify->notify("%s", epee::string_tools::pod_to_hex(id).c_str(), NULL); return true; } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_weight_limit() +bool Blockchain::prune_blockchain(uint32_t pruning_seed) +{ + uint8_t hf_version = m_hardfork->get_current_version(); + if (hf_version < 10) + { + MERROR("Most of the network will only be ready for pruned blockchains from v10, not pruning"); + return false; + } + return m_db->prune_blockchain(pruning_seed); +} +//------------------------------------------------------------------ +bool Blockchain::update_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->update_pruning(); +} +//------------------------------------------------------------------ +bool Blockchain::check_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->check_pruning(); +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) const { - uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); + PERF_TIMER(get_next_long_term_block_weight); + + const uint64_t db_height = m_db->height(); + const uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); + + const uint8_t hf_version = get_current_hard_fork_version(); + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + return block_weight; + + std::vector<uint64_t> weights; + weights.resize(nblocks); + for (uint64_t h = 0; h < nblocks; ++h) + weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h); + uint64_t long_term_median = epee::misc_utils::median(weights); + uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = long_term_effective_median_block_weight + long_term_effective_median_block_weight * 2 / 5; + uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint); + + return long_term_block_weight; +} +//------------------------------------------------------------------ +bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight) +{ + PERF_TIMER(update_next_cumulative_weight_limit); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector<size_t> weights; - get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(weights); - m_current_block_cumul_weight_median = median; - if(median <= full_reward_zone) - median = full_reward_zone; + // when we reach this, the last hf version is not yet written to the db + const uint64_t db_height = m_db->height(); + const uint8_t hf_version = get_current_hard_fork_version(); + uint64_t full_reward_zone = get_min_block_weight(hf_version); + uint64_t long_term_block_weight; + + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + { + std::vector<uint64_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + m_current_block_cumul_weight_median = epee::misc_utils::median(weights); + long_term_block_weight = weights.back(); + } + else + { + const uint64_t block_weight = m_db->get_block_weight(db_height - 1); + + std::vector<uint64_t> weights, new_weights; + uint64_t long_term_median; + if (db_height == 1) + { + long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5; + } + else + { + uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); + if (nblocks == db_height) + --nblocks; + weights.resize(nblocks); + for (uint64_t h = 0; h < nblocks; ++h) + weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h - 1); + new_weights = weights; + long_term_median = epee::misc_utils::median(weights); + } + + m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = m_long_term_effective_median_block_weight + m_long_term_effective_median_block_weight * 2 / 5; + long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint); + + if (new_weights.empty()) + new_weights.resize(1); + new_weights[0] = long_term_block_weight; + long_term_median = epee::misc_utils::median(new_weights); + m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + short_term_constraint = m_long_term_effective_median_block_weight + m_long_term_effective_median_block_weight * 2 / 5; + + weights.clear(); + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + + uint64_t short_term_median = epee::misc_utils::median(weights); + uint64_t effective_median_block_weight = std::min<uint64_t>(std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, short_term_median), CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR * m_long_term_effective_median_block_weight); + + m_current_block_cumul_weight_median = effective_median_block_weight; + } + + if (m_current_block_cumul_weight_median <= full_reward_zone) + m_current_block_cumul_weight_median = full_reward_zone; + + m_current_block_cumul_weight_limit = m_current_block_cumul_weight_median * 2; + + if (long_term_effective_median_block_weight) + *long_term_effective_median_block_weight = m_long_term_effective_median_block_weight; - m_current_block_cumul_weight_limit = median*2; return true; } //------------------------------------------------------------------ -bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc) +bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc) { LOG_PRINT_L3("Blockchain::" << __func__); - //copy block here to let modify block.target - block bl = bl_; crypto::hash id = get_block_hash(bl); CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process CRITICAL_REGION_LOCAL1(m_blockchain_lock); @@ -3685,10 +3930,11 @@ void Blockchain::check_against_checkpoints(const checkpoints& points, bool enfor CRITICAL_REGION_LOCAL(m_blockchain_lock); stop_batch = m_db->batch_start(); + const uint64_t blockchain_height = m_db->height(); for (const auto& pt : pts) { // if the checkpoint is for a block we don't have yet, move on - if (pt.first >= m_db->height()) + if (pt.first >= blockchain_height) { continue; } @@ -3838,37 +4084,17 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) CRITICAL_REGION_END(); m_tx_pool.unlock(); + update_blockchain_pruning(); + return success; } //------------------------------------------------------------------ -void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, const std::vector<output_data_t> &extra_tx_map) const +void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) const { try { m_db->get_output_key(epee::span<const uint64_t>(&amount, 1), offsets, outputs, true); - if (outputs.size() < offsets.size()) - { - const uint64_t n_outputs = m_db->get_num_outputs(amount); - for (size_t i = outputs.size(); i < offsets.size(); ++i) - { - uint64_t idx = offsets[i]; - if (idx < n_outputs) - { - MWARNING("Index " << idx << " not found in db for amount " << amount << ", but it is less than the number of entries"); - break; - } - else if (idx < n_outputs + extra_tx_map.size()) - { - outputs.push_back(extra_tx_map[idx - n_outputs]); - } - else - { - MWARNING("missed " << amount << "/" << idx << " in " << extra_tx_map.size() << " (chain " << n_outputs << ")"); - break; - } - } - } } catch (const std::exception& e) { @@ -3983,41 +4209,14 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector // vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries // and is threaded if possible. The table (m_scan_table) will be used later when querying output // keys. -static bool update_output_map(std::map<uint64_t, std::vector<output_data_t>> &extra_tx_map, const transaction &tx, uint64_t height, bool miner) -{ - MTRACE("Blockchain::" << __func__); - for (size_t i = 0; i < tx.vout.size(); ++i) - { - const auto &out = tx.vout[i]; - if (out.target.type() != typeid(txout_to_key)) - continue; - const txout_to_key &out_to_key = boost::get<txout_to_key>(out.target); - rct::key commitment; - uint64_t amount = out.amount; - if (miner && tx.version == 2) - { - commitment = rct::zeroCommit(amount); - amount = 0; - } - else if (tx.version > 1) - { - CHECK_AND_ASSERT_MES(i < tx.rct_signatures.outPk.size(), false, "Invalid outPk size"); - commitment = tx.rct_signatures.outPk[i].mask; - } - else - commitment = rct::zero(); - extra_tx_map[amount].push_back(output_data_t{out_to_key.key, tx.unlock_time, height, commitment}); - } - return true; -} - -bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry) +bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry, std::vector<block> &blocks) { MTRACE("Blockchain::" << __func__); TIME_MEASURE_START(prepare); bool stop_batch; uint64_t bytes = 0; size_t total_txs = 0; + blocks.clear(); // Order of locking must be: // m_incoming_tx_lock (optional) @@ -4064,7 +4263,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete bool blocks_exist = false; tools::threadpool& tpool = tools::threadpool::getInstance(); unsigned threads = tpool.get_max_concurrency(); - std::vector<block> blocks; blocks.resize(blocks_entry.size()); if (1) @@ -4080,6 +4278,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete auto it = blocks_entry.begin(); unsigned blockidx = 0; + const crypto::hash tophash = m_db->top_block_hash(); for (unsigned i = 0; i < threads; i++) { for (unsigned int j = 0; j < batches; j++, ++blockidx) @@ -4092,18 +4291,15 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete // check first block and skip all blocks if its not chained properly if (blockidx == 0) { - crypto::hash tophash = m_db->top_block_hash(); if (block.prev_id != tophash) { MDEBUG("Skipping prepare blocks. New blocks don't belong to chain."); + blocks.clear(); return true; } } if (have_block(get_block_hash(block))) - { blocks_exist = true; - break; - } std::advance(it, 1); } @@ -4117,10 +4313,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete return false; if (have_block(get_block_hash(block))) - { blocks_exist = true; - break; - } std::advance(it, 1); } @@ -4135,7 +4328,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete unsigned nblocks = batches; if (i < extra) ++nblocks; - tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, epee::span<const block>(&blocks[i], nblocks), std::ref(maps[i])), true); + tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, epee::span<const block>(&blocks[thread_height - height], nblocks), std::ref(maps[i])), true); thread_height += nblocks; } @@ -4156,7 +4349,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete if (blocks_exist) { - MDEBUG("Skipping prepare blocks. Blocks exist."); + MDEBUG("Skipping remainder of prepare blocks. Blocks exist."); return true; } @@ -4179,7 +4372,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete // [input] stores all absolute_offsets for each amount std::map<uint64_t, std::vector<uint64_t>> offset_map; // [output] stores all output_data_t for each absolute_offset - std::map<uint64_t, std::vector<output_data_t>> tx_map, extra_tx_map; + std::map<uint64_t, std::vector<output_data_t>> tx_map; std::vector<std::pair<cryptonote::transaction, crypto::hash>> txes(total_txs); #define SCAN_TABLE_QUIT(m) \ @@ -4196,8 +4389,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete if (m_cancel) return false; - if (!update_output_map(extra_tx_map, blocks[block_index].miner_tx, height + block_index, true)) - SCAN_TABLE_QUIT("Error building extra tx map."); for (const auto &tx_blob : entry.txs) { if (tx_index >= txes.size()) @@ -4256,8 +4447,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete offset_map[in_to_key.amount].push_back(offset); } - if (!update_output_map(extra_tx_map, tx, height + block_index, false)) - SCAN_TABLE_QUIT("Error building extra tx map."); } ++block_index; } @@ -4282,7 +4471,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::cref(extra_tx_map[amount])), true); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount])), true); } waiter.wait(&tpool); } @@ -4291,7 +4480,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - output_scan_worker(amount, offset_map[amount], tx_map[amount], extra_tx_map[amount]); + output_scan_worker(amount, offset_map[amount], tx_map[amount]); } } @@ -4516,7 +4705,7 @@ void Blockchain::cancel() } #if defined(PER_BLOCK_CHECKPOINT) -static const char expected_block_hashes_hash[] = "954cb2bbfa2fe6f74b2cdd22a1a4c767aea249ad47ad4f7c9445f0f03260f511"; +static const char expected_block_hashes_hash[] = "570ce2357b08fadac6058e34f95c5e08323f9325de260d07b091a281a948a7b0"; void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints) { if (get_checkpoints == nullptr || !m_fast_sync) @@ -4589,10 +4778,11 @@ void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get uint64_t fee; bool relayed, do_not_relay, double_spend_seen; transaction pool_tx; + blobdata txblob; for(const transaction &tx : txs) { crypto::hash tx_hash = get_transaction_hash(tx); - m_tx_pool.take_tx(tx_hash, pool_tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen); + m_tx_pool.take_tx(tx_hash, pool_tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen); } } } @@ -4665,4 +4855,5 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; +template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 67bccc6c6..4744defc5 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -37,6 +37,7 @@ #include <boost/multi_index/global_fun.hpp> #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/member.hpp> +#include <boost/circular_buffer.hpp> #include <atomic> #include <functional> #include <unordered_map> @@ -229,11 +230,12 @@ namespace cryptonote /** * @brief performs some preprocessing on a group of incoming blocks to speed up verification * - * @param blocks a list of incoming blocks + * @param blocks_entry a list of incoming blocks + * @param blocks the parsed blocks * * @return false on erroneous blocks, else true */ - bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry, std::vector<block> &blocks); /** * @brief incoming blocks post-processing, cleanup, and disk sync @@ -521,10 +523,12 @@ namespace cryptonote * * @param tx_id the hash of the transaction to fetch indices for * @param indexs return-by-reference the global indices for the transaction's outputs + * @param n_txes how many txes in a row to get results for * * @return false if the transaction does not exist, or if no indices are found, otherwise true */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const; /** * @brief stores the blockchain @@ -629,6 +633,13 @@ namespace cryptonote uint64_t get_current_cumulative_block_weight_limit() const; /** + * @brief gets the long term block weight for a new block + * + * @return the long term block weight + */ + uint64_t get_next_long_term_block_weight(uint64_t block_weight) const; + + /** * @brief gets the block weight median based on recent blocks (same window as for the limit) * * @return the median @@ -675,6 +686,8 @@ namespace cryptonote template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; template<class t_ids_container, class t_tx_container, class t_missed_container> + bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; //debug functions @@ -727,11 +740,18 @@ namespace cryptonote /** * @brief sets a block notify object to call for every new block * - * @param notify the notify object to cal at every new block + * @param notify the notify object to call at every new block */ void set_block_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_block_notify = notify; } /** + * @brief sets a reorg notify object to call for every reorg + * + * @param notify the notify object to call at every reorg + */ + void set_reorg_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_reorg_notify = notify; } + + /** * @brief Put DB in safe sync mode */ void safesyncmode(const bool onoff); @@ -923,7 +943,7 @@ namespace cryptonote * @param outputs return-by-reference the outputs collected */ void output_scan_worker(const uint64_t amount,const std::vector<uint64_t> &offsets, - std::vector<output_data_t> &outputs, const std::vector<output_data_t> &extra_tx_map) const; + std::vector<output_data_t> &outputs) const; /** * @brief computes the "short" and "long" hashes for a set of blocks @@ -954,6 +974,10 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); + uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); } + bool prune_blockchain(uint32_t pruning_seed = 0); + bool update_blockchain_pruning(); + bool check_blockchain_pruning(); void lock(); void unlock(); @@ -979,7 +1003,9 @@ namespace cryptonote */ void pop_blocks(uint64_t nblocks); +#ifndef IN_UNIT_TESTS private: +#endif // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index; @@ -1032,6 +1058,8 @@ namespace cryptonote std::vector<uint64_t> m_timestamps; std::vector<difficulty_type> m_difficulties; uint64_t m_timestamps_and_difficulties_height; + uint64_t m_long_term_block_weights_window; + uint64_t m_long_term_effective_median_block_weight; epee::critical_section m_difficulty_lock; crypto::hash m_difficulty_for_next_block_top_hash; @@ -1069,6 +1097,7 @@ namespace cryptonote bool m_btc_valid; std::shared_ptr<tools::Notify> m_block_notify; + std::shared_ptr<tools::Notify> m_reorg_notify; /** * @brief collects the keys for all outputs being "spent" as an input @@ -1264,7 +1293,7 @@ namespace cryptonote * @param sz return-by-reference the list of weights * @param count the number of blocks to get weights for */ - void get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const; + void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) @@ -1363,10 +1392,12 @@ namespace cryptonote /** * @brief calculate the block weight limit for the next block to be added * + * @param long_term_effective_median_block_weight optionally return that value + * * @return true */ - bool update_next_cumulative_weight_limit(); - void return_tx_to_pool(std::vector<transaction> &txs); + bool update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight = NULL); + void return_tx_to_pool(std::vector<std::pair<transaction, blobdata>> &txs); /** * @brief make sure a transaction isn't attempting a double-spend diff --git a/src/cryptonote_core/blockchain_storage_boost_serialization.h b/src/cryptonote_core/blockchain_storage_boost_serialization.h index f4f9f20ca..cfeb5acfc 100644 --- a/src/cryptonote_core/blockchain_storage_boost_serialization.h +++ b/src/cryptonote_core/blockchain_storage_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index f3249ea92..58acdb6bb 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -105,6 +105,11 @@ namespace cryptonote "disable-dns-checkpoints" , "Do not retrieve checkpoints from DNS" }; + const command_line::arg_descriptor<size_t> arg_block_download_max_size = { + "block-download-max-size" + , "Set maximum size of block download queue in bytes (0 for default)" + , 0 + }; static const command_line::arg_descriptor<bool> arg_test_drop_download = { "test-drop-download" @@ -175,6 +180,31 @@ namespace cryptonote , "Run a program for each new block, '%s' will be replaced by the block hash" , "" }; + static const command_line::arg_descriptor<bool> arg_prune_blockchain = { + "prune-blockchain" + , "Prune blockchain" + , false + }; + static const command_line::arg_descriptor<std::string> arg_reorg_notify = { + "reorg-notify" + , "Run a program for each reorg, '%s' will be replaced by the split height, " + "'%h' will be replaced by the new blockchain height, '%n' will be " + "replaced by the number of new blocks in the new chain, and '%d' will be " + "replaced by the number of blocks discarded from the old chain" + , "" + }; + static const command_line::arg_descriptor<std::string> arg_block_rate_notify = { + "block-rate-notify" + , "Run a program when the block rate undergoes large fluctuations. This might " + "be a sign of large amounts of hash rate going on and off the Monero network, " + "and thus be of potential interest in predicting attacks. %t will be replaced " + "by the number of minutes for the observation window, %b by the number of " + "blocks observed within that window, and %e by the number of blocks that was " + "expected in that window. It is suggested that this notification is used to " + "automatically increase the number of confirmations required before a payment " + "is acted upon." + , "" + }; //----------------------------------------------------------------------------------------------- core::core(i_cryptonote_protocol* pprotocol): @@ -285,9 +315,13 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); + command_line::add_arg(desc, arg_block_download_max_size); command_line::add_arg(desc, arg_max_txpool_weight); command_line::add_arg(desc, arg_pad_transactions); command_line::add_arg(desc, arg_block_notify); + command_line::add_arg(desc, arg_prune_blockchain); + command_line::add_arg(desc, arg_reorg_notify); + command_line::add_arg(desc, arg_block_rate_notify); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -374,6 +408,11 @@ namespace cryptonote return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- + bool core::get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const + { + return m_blockchain_storage.get_split_transactions_blobs(txs_ids, txs, missed_txs); + } + //----------------------------------------------------------------------------------------------- bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const { m_mempool.get_transaction_backlog(backlog); @@ -413,6 +452,7 @@ namespace cryptonote uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); + bool prune_blockchain = command_line::get_arg(vm, arg_prune_blockchain); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -561,12 +601,33 @@ namespace cryptonote } catch (const std::exception &e) { - MERROR("Failed to parse block notify spec"); + MERROR("Failed to parse block notify spec: " << e.what()); + } + + try + { + if (!command_line::is_arg_defaulted(vm, arg_reorg_notify)) + m_blockchain_storage.set_reorg_notify(std::shared_ptr<tools::Notify>(new tools::Notify(command_line::get_arg(vm, arg_reorg_notify).c_str()))); + } + catch (const std::exception &e) + { + MERROR("Failed to parse reorg notify spec: " << e.what()); + } + + try + { + if (!command_line::is_arg_defaulted(vm, arg_block_rate_notify)) + m_block_rate_notify.reset(new tools::Notify(command_line::get_arg(vm, arg_block_rate_notify).c_str())); + } + catch (const std::exception &e) + { + MERROR("Failed to parse block rate notify spec: " << e.what()); } const std::pair<uint8_t, uint64_t> regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; const cryptonote::test_options regtest_test_options = { - regtest_hard_forks + regtest_hard_forks, + 0 }; const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty, get_checkpoints); @@ -607,6 +668,14 @@ namespace cryptonote r = m_miner.init(vm, m_nettype); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize miner instance"); + if (prune_blockchain) + { + // display a message if the blockchain is not pruned yet + if (m_blockchain_storage.get_current_blockchain_height() > 1 && !m_blockchain_storage.get_blockchain_pruning_seed()) + MGINFO("Pruning blockchain..."); + CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + } + return load_state_data(); } //----------------------------------------------------------------------------------------------- @@ -655,7 +724,7 @@ namespace cryptonote return false; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) { tvc = boost::value_initialized<tx_verification_context>(); @@ -668,9 +737,8 @@ namespace cryptonote } tx_hash = crypto::null_hash; - tx_prefixt_hash = crypto::null_hash; - if(!parse_tx_from_blob(tx, tx_hash, tx_prefixt_hash, tx_blob)) + if(!parse_tx_from_blob(tx, tx_hash, tx_blob)) { LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to parse, rejected"); tvc.m_verifivation_failed = true; @@ -703,7 +771,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) { if(!check_tx_syntax(tx)) { @@ -790,6 +858,7 @@ namespace cryptonote } break; case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) { MERROR_VER("Bulletproof does not have canonical form"); @@ -817,7 +886,7 @@ namespace cryptonote { if (!tx_info[n].result) continue; - if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof) + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof && tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof2) continue; if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) { @@ -836,7 +905,7 @@ namespace cryptonote TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); - struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; }; + struct result { bool res; cryptonote::transaction tx; crypto::hash hash; }; std::vector<result> results(tx_blobs.size()); tvc.resize(tx_blobs.size()); @@ -847,7 +916,7 @@ namespace cryptonote tpool.submit(&waiter, [&, i, it] { try { - results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); + results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, keeped_by_block, relayed, do_not_relay); } catch (const std::exception &e) { @@ -877,7 +946,7 @@ namespace cryptonote tpool.submit(&waiter, [&, i, it] { try { - results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); + results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, keeped_by_block, relayed, do_not_relay); } catch (const std::exception &e) { @@ -913,7 +982,7 @@ namespace cryptonote continue; const size_t weight = get_transaction_weight(results[i].tx, it->size()); - ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -1127,11 +1196,10 @@ namespace cryptonote bool core::add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { crypto::hash tx_hash = get_transaction_hash(tx); - crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); blobdata bl; t_serializable_object_to_blob(tx, bl); size_t tx_weight = get_transaction_weight(tx, bl.size()); - return add_new_tx(tx, tx_hash, bl, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); + return add_new_tx(tx, tx_hash, bl, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -1139,7 +1207,7 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if(m_mempool.have_tx(tx_hash)) { @@ -1180,8 +1248,8 @@ namespace cryptonote { std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) + crypto::hash tx_hash; + if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash)) { LOG_ERROR("Failed to parse relayed transaction"); return; @@ -1220,6 +1288,11 @@ namespace cryptonote return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); } //----------------------------------------------------------------------------------------------- + bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const + { + return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, n_txes, indexs); + } + //----------------------------------------------------------------------------------------------- void core::pause_mine() { m_miner.pause(); @@ -1257,7 +1330,13 @@ namespace cryptonote m_miner.resume(); return false; } - prepare_handle_incoming_blocks(blocks); + std::vector<block> pblocks; + if (!prepare_handle_incoming_blocks(blocks, pblocks)) + { + MERROR("Block found, but failed to prepare to add"); + m_miner.resume(); + return false; + } m_blockchain_storage.add_new_block(b, bvc); cleanup_handle_incoming_blocks(true); //anyway - update miner template @@ -1308,10 +1387,14 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks) + bool core::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry, std::vector<block> &blocks) { m_incoming_tx_lock.lock(); - m_blockchain_storage.prepare_handle_incoming_blocks(blocks); + if (!m_blockchain_storage.prepare_handle_incoming_blocks(blocks_entry, blocks)) + { + cleanup_handle_incoming_blocks(false); + return false; + } return true; } @@ -1328,7 +1411,7 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate) + bool core::handle_incoming_block(const blobdata& block_blob, const block *b, block_verification_context& bvc, bool update_miner_blocktemplate) { TRY_ENTRY(); @@ -1344,14 +1427,18 @@ namespace cryptonote return false; } - block b = AUTO_VAL_INIT(b); - if(!parse_and_validate_block_from_blob(block_blob, b)) + block lb; + if (!b) { - LOG_PRINT_L1("Failed to parse and validate new block"); - bvc.m_verifivation_failed = true; - return false; + if(!parse_and_validate_block_from_blob(block_blob, lb)) + { + LOG_PRINT_L1("Failed to parse and validate new block"); + bvc.m_verifivation_failed = true; + return false; + } + b = &lb; } - add_new_block(b, bvc); + add_new_block(*b, bvc); if(update_miner_blocktemplate && bvc.m_added_to_main_chain) update_miner_block_template(); return true; @@ -1391,9 +1478,9 @@ namespace cryptonote return m_blockchain_storage.have_block(id); } //----------------------------------------------------------------------------------------------- - bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob) const + bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const { - return parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash); + return parse_and_validate_tx_from_blob(blob, tx, tx_hash); } //----------------------------------------------------------------------------------------------- bool core::check_tx_syntax(const transaction& tx) const @@ -1496,6 +1583,7 @@ namespace cryptonote m_check_updates_interval.do_call(boost::bind(&core::check_updates, this)); m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this)); m_block_rate_interval.do_call(boost::bind(&core::check_block_rate, this)); + m_blockchain_pruning_interval.do_call(boost::bind(&core::update_blockchain_pruning, this)); m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -1713,7 +1801,7 @@ namespace cryptonote const time_t now = time(NULL); const std::vector<time_t> timestamps = m_blockchain_storage.get_last_block_timestamps(60); - static const unsigned int seconds[] = { 5400, 1800, 600 }; + static const unsigned int seconds[] = { 5400, 3600, 1800, 1200, 600 }; for (size_t n = 0; n < sizeof(seconds)/sizeof(seconds[0]); ++n) { unsigned int b = 0; @@ -1724,6 +1812,14 @@ namespace cryptonote if (p < threshold) { MWARNING("There were " << b << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Monero network or under attack. Or it could be just sheer bad luck."); + + std::shared_ptr<tools::Notify> block_rate_notify = m_block_rate_notify; + if (block_rate_notify) + { + auto expected = seconds[n] / DIFFICULTY_TARGET_V2; + block_rate_notify->notify("%t", std::to_string(seconds[n] / 60).c_str(), "%b", std::to_string(b).c_str(), "%e", std::to_string(expected).c_str(), NULL); + } + break; // no need to look further } } @@ -1731,6 +1827,16 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::update_blockchain_pruning() + { + return m_blockchain_storage.update_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- + bool core::check_blockchain_pruning() + { + return m_blockchain_storage.check_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- void core::set_target_blockchain_height(uint64_t target_blockchain_height) { m_target_blockchain_height = target_blockchain_height; @@ -1753,6 +1859,16 @@ namespace cryptonote return si.available; } //----------------------------------------------------------------------------------------------- + uint32_t core::get_blockchain_pruning_seed() const + { + return get_blockchain_storage().get_blockchain_pruning_seed(); + } + //----------------------------------------------------------------------------------------------- + bool core::prune_blockchain(uint32_t pruning_seed) + { + return get_blockchain_storage().prune_blockchain(pruning_seed); + } + //----------------------------------------------------------------------------------------------- std::time_t core::get_start_time() const { return start_time; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index cc53fce58..356265dd6 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -54,6 +54,7 @@ namespace cryptonote { struct test_options { const std::pair<uint8_t, uint64_t> *hard_forks; + const size_t long_term_block_weight_window; }; extern const command_line::arg_descriptor<std::string, false, true, 2> arg_data_dir; @@ -62,6 +63,7 @@ namespace cryptonote extern const command_line::arg_descriptor<bool, false> arg_regtest_on; extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty; extern const command_line::arg_descriptor<bool> arg_offline; + extern const command_line::arg_descriptor<size_t> arg_block_download_max_size; /************************************************************************/ /* */ @@ -144,20 +146,21 @@ namespace cryptonote * optionally updates the miner's block template. * * @param block_blob the block to be added + * @param block the block to be added, or NULL * @param bvc return-by-reference metadata context about the block's validity * @param update_miner_blocktemplate whether or not to update the miner's block template * * @return false if loading new checkpoints fails, or the block is not * added, otherwise true */ - bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true); + bool handle_incoming_block(const blobdata& block_blob, const block *b, block_verification_context& bvc, bool update_miner_blocktemplate = true); /** * @copydoc Blockchain::prepare_handle_incoming_blocks * * @note see Blockchain::prepare_handle_incoming_blocks */ - bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry, std::vector<block> &blocks); /** * @copydoc Blockchain::cleanup_handle_incoming_blocks @@ -359,6 +362,13 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ + bool get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const; + + /** + * @copydoc Blockchain::get_transactions + * + * @note see Blockchain::get_transactions + */ bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; /** @@ -534,6 +544,7 @@ namespace cryptonote * @note see Blockchain::get_tx_outputs_gindexs */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const; /** * @copydoc Blockchain::get_tail_id @@ -782,6 +793,36 @@ namespace cryptonote */ bool offline() const { return m_offline; } + /** + * @brief get the blockchain pruning seed + * + * @return the blockchain pruning seed + */ + uint32_t get_blockchain_pruning_seed() const; + + /** + * @brief prune the blockchain + * + * @param pruning_seed the seed to use to prune the chain (0 for default, highly recommended) + * + * @return true iff success + */ + bool prune_blockchain(uint32_t pruning_seed = 0); + + /** + * @brief incrementally prunes blockchain + * + * @return true on success, false otherwise + */ + bool update_blockchain_pruning(); + + /** + * @brief checks the blockchain pruning if enabled + * + * @return true on success, false otherwise + */ + bool check_blockchain_pruning(); + private: /** @@ -789,13 +830,12 @@ namespace cryptonote * * @param tx_hash the transaction's hash * @param blob the transaction as a blob - * @param tx_prefix_hash the transaction prefix' hash * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -835,7 +875,7 @@ namespace cryptonote * * @note see parse_tx_from_blob(transaction&, crypto::hash&, crypto::hash&, const blobdata&) const */ - bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob) const; + bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const; /** * @brief check a transaction's syntax @@ -868,8 +908,8 @@ namespace cryptonote bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; void set_semantics_failed(const crypto::hash &tx_hash); - bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block); @@ -984,6 +1024,7 @@ namespace cryptonote epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions epee::math_helper::once_a_time_seconds<60*10, true> m_check_disk_space_interval; //!< interval for checking for disk space epee::math_helper::once_a_time_seconds<90, false> m_block_rate_interval; //!< interval for checking block rate + epee::math_helper::once_a_time_seconds<60*60*5, true> m_blockchain_pruning_interval; //!< interval for incremental blockchain pruning std::atomic<bool> m_starter_message_showed; //!< has the "daemon will sync now" message been shown? @@ -1021,6 +1062,8 @@ namespace cryptonote bool m_fluffy_blocks_enabled; bool m_offline; bool m_pad_transactions; + + std::shared_ptr<tools::Notify> m_block_rate_notify; }; } diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index f443d638c..854f3d1c5 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -198,7 +198,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -226,13 +226,15 @@ namespace cryptonote std::vector<tx_extra_field> tx_extra_fields; if (parse_tx_extra(tx.extra, tx_extra_fields)) { + bool add_dummy_payment_id = true; tx_extra_nonce extra_nonce; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash8 payment_id = null_hash8; - if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + crypto::hash payment_id = null_hash; + crypto::hash8 payment_id8 = null_hash8; + if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) { - LOG_PRINT_L2("Encrypting payment id " << payment_id); + LOG_PRINT_L2("Encrypting payment id " << payment_id8); crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); if (view_key_pub == null_pkey) { @@ -240,28 +242,60 @@ namespace cryptonote return false; } - if (!hwdev.encrypt_payment_id(payment_id, view_key_pub, tx_key)) + if (!hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key)) { LOG_ERROR("Failed to encrypt payment id"); return false; } std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); remove_field_from_tx_extra(tx.extra, typeid(tx_extra_nonce)); if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) { LOG_ERROR("Failed to add encrypted payment id to tx extra"); return false; } - LOG_PRINT_L1("Encrypted payment ID: " << payment_id); + LOG_PRINT_L1("Encrypted payment ID: " << payment_id8); + add_dummy_payment_id = false; + } + else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + add_dummy_payment_id = false; + } + } + + // we don't add one if we've got more than the usual 1 destination plus change + if (destinations.size() > 2) + add_dummy_payment_id = false; + + if (add_dummy_payment_id) + { + // if we have neither long nor short payment id, add a dummy short one, + // this should end up being the vast majority of txes as time goes on + std::string extra_nonce; + crypto::hash8 payment_id8 = null_hash8; + crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); + if (view_key_pub == null_pkey) + { + LOG_ERROR("Failed to get key to encrypt dummy payment id with"); + } + else + { + hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); + if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) + { + LOG_ERROR("Failed to add dummy encrypted payment id to tx extra"); + // continue anyway + } } } } else { - LOG_ERROR("Failed to parse tx extra"); - return false; + MWARNING("Failed to parse tx extra"); + tx_extra_fields.clear(); } struct input_generation_context_data @@ -371,49 +405,12 @@ namespace cryptonote for(const tx_destination_entry& dst_entr: destinations) { CHECK_AND_ASSERT_MES(dst_entr.amount > 0 || tx.version > 1, false, "Destination with wrong amount: " << dst_entr.amount); - crypto::key_derivation derivation; crypto::public_key out_eph_public_key; - // make additional tx pubkey if necessary - keypair additional_txkey; - if (need_additional_txkeys) - { - additional_txkey.sec = additional_tx_keys[output_index]; - if (dst_entr.is_subaddress) - additional_txkey.pub = rct::rct2pk(hwdev.scalarmultKey(rct::pk2rct(dst_entr.addr.m_spend_public_key), rct::sk2rct(additional_txkey.sec))); - else - additional_txkey.pub = rct::rct2pk(hwdev.scalarmultBase(rct::sk2rct(additional_txkey.sec))); - } - - bool r; - if (change_addr && dst_entr.addr == *change_addr) - { - // sending change to yourself; derivation = a*R - r = hwdev.generate_key_derivation(txkey_pub, sender_account_keys.m_view_secret_key, derivation); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << txkey_pub << ", " << sender_account_keys.m_view_secret_key << ")"); - } - else - { - // sending to the recipient; derivation = r*A (or s*C in the subaddress scheme) - r = hwdev.generate_key_derivation(dst_entr.addr.m_view_public_key, dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key, derivation); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << dst_entr.addr.m_view_public_key << ", " << (dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key) << ")"); - } - - if (need_additional_txkeys) - { - additional_tx_public_keys.push_back(additional_txkey.pub); - } - - if (tx.version > 1) - { - crypto::secret_key scalar1; - hwdev.derivation_to_scalar(derivation, output_index, scalar1); - amount_keys.push_back(rct::sk2rct(scalar1)); - } - r = hwdev.derive_public_key(derivation, output_index, dst_entr.addr.m_spend_public_key, out_eph_public_key); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << output_index << ", "<< dst_entr.addr.m_spend_public_key << ")"); - - hwdev.add_output_key_mapping(dst_entr.addr.m_view_public_key, dst_entr.addr.m_spend_public_key, dst_entr.is_subaddress, output_index, amount_keys.back(), out_eph_public_key); + hwdev.generate_output_ephemeral_keys(tx.version,sender_account_keys, txkey_pub, tx_key, + dst_entr, change_addr, output_index, + need_additional_txkeys, additional_tx_keys, + additional_tx_public_keys, amount_keys, out_eph_public_key); tx_out out; out.amount = dst_entr.amount; @@ -497,7 +494,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1 || range_proof_type != rct::RangeProofBorromean; + bool use_simple_rct = sources.size() > 1 || rct_config.range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -595,9 +592,9 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, range_proof_type, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, rct_config, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, rct_config, hwdev); // same index assumption memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -610,7 +607,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -628,7 +625,7 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, range_proof_type, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, msout); hwdev.close_tx(); return r; } @@ -640,7 +637,7 @@ namespace cryptonote crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0}, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 87edafe9d..cb1561c2d 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -73,25 +73,36 @@ namespace cryptonote struct tx_destination_entry { + std::string original; uint64_t amount; //money account_public_address addr; //destination address bool is_subaddress; + bool is_integrated; - tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false) { } - tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress) { } + tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false), is_integrated(false) { } + tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } + tx_destination_entry(const std::string &o, uint64_t a, const account_public_address &ad, bool is_subaddress) : original(o), amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } BEGIN_SERIALIZE_OBJECT() + FIELD(original) VARINT_FIELD(amount) FIELD(addr) FIELD(is_subaddress) + FIELD(is_integrated) END_SERIALIZE() }; //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL); + bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys, + std::vector<crypto::public_key> &additional_tx_public_keys, + std::vector<rct::key> &amount_keys, + crypto::public_key &out_eph_public_key) ; bool generate_genesis_block( block& bl @@ -102,7 +113,7 @@ namespace cryptonote } BOOST_CLASS_VERSION(cryptonote::tx_source_entry, 1) -BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 1) +BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 2) namespace boost { @@ -132,6 +143,13 @@ namespace boost if (ver < 1) return; a & x.is_subaddress; + if (ver < 2) + { + x.is_integrated = false; + return; + } + a & x.original; + a & x.is_integrated; } } } diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 1c8f1c62c..c1cbe2acd 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -453,7 +453,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) + bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -469,7 +469,7 @@ namespace cryptonote MERROR("Failed to find tx in txpool"); return false; } - cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(id); + txblob = m_blockchain.get_txpool_tx_blob(id); auto ci = m_parsed_tx_cache.find(id); if (ci != m_parsed_tx_cache.end()) { diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 670d70d77..877f2b82f 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -61,7 +61,7 @@ namespace cryptonote class txCompare { public: - bool operator()(const tx_by_fee_and_receive_time_entry& a, const tx_by_fee_and_receive_time_entry& b) + bool operator()(const tx_by_fee_and_receive_time_entry& a, const tx_by_fee_and_receive_time_entry& b) const { // sort by greatest first, not least if (a.first.first > b.first.first) return true; @@ -133,6 +133,7 @@ namespace cryptonote * * @param id the hash of the transaction * @param tx return-by-reference the transaction taken + * @param txblob return-by-reference the transaction as a blob * @param tx_weight return-by-reference the transaction's weight * @param fee the transaction fee * @param relayed return-by-reference was transaction relayed to us by the network? @@ -141,7 +142,7 @@ namespace cryptonote * * @return true unless the transaction cannot be found in the pool */ - bool take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); + bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); /** * @brief checks if the pool has a transaction with the given hash diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt index 1189ccf22..bfcf42767 100644 --- a/src/cryptonote_protocol/CMakeLists.txt +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 05f4189fb..b4f9daa74 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -31,8 +31,10 @@ #include <vector> #include <unordered_map> #include <boost/uuid/nil_generator.hpp> +#include <boost/uuid/uuid_io.hpp> #include "string_tools.h" #include "cryptonote_protocol_defs.h" +#include "common/pruning.h" #include "block_queue.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -59,7 +61,10 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp if (has_hashes) { for (const crypto::hash &h: hashes) + { requested_hashes.insert(h); + have_blocks.insert(h); + } set_span_hashes(height, connection_id, hashes); } } @@ -89,7 +94,10 @@ void block_queue::erase_block(block_map::iterator j) { CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); for (const crypto::hash &h: j->hashes) + { requested_hashes.erase(h); + have_blocks.erase(h); + } blocks.erase(j); } @@ -97,12 +105,10 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con { boost::unique_lock<boost::recursive_mutex> lock(mutex); block_map::iterator i = blocks.begin(); - if (i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) { block_map::iterator j = i++; - if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) + if (j->blocks.empty() && live_connections.find(j->connection_id) == live_connections.end()) { erase_block(j); } @@ -151,23 +157,56 @@ uint64_t block_queue::get_max_block_height() const return height; } +uint64_t block_queue::get_next_needed_height(uint64_t blockchain_height) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return blockchain_height; + uint64_t last_needed_height = blockchain_height; + bool first = true; + for (const auto &span: blocks) + { + if (span.start_block_height + span.nblocks - 1 < blockchain_height) + continue; + if (span.start_block_height != last_needed_height || (first && span.blocks.empty())) + return last_needed_height; + last_needed_height = span.start_block_height + span.nblocks; + first = false; + } + return last_needed_height; +} + void block_queue::print() const { boost::unique_lock<boost::recursive_mutex> lock(mutex); MDEBUG("Block queue has " << blocks.size() << " spans"); for (const auto &span: blocks) - MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (is_blockchain_placeholder(span) ? "blockchain" : span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); + MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); } -std::string block_queue::get_overview() const +std::string block_queue::get_overview(uint64_t blockchain_height) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return "[]"; block_map::const_iterator i = blocks.begin(); - std::string s = std::string("[") + std::to_string(i->start_block_height + i->nblocks - 1) + ":"; - while (++i != blocks.end()) - s += i->blocks.empty() ? "." : "o"; + std::string s = std::string("["); + uint64_t expected = blockchain_height; + while (i != blocks.end()) + { + if (expected > i->start_block_height) + { + s += "<"; + } + else + { + if (expected < i->start_block_height) + s += std::string(std::max((uint64_t)1, (i->start_block_height - expected) / (i->nblocks ? i->nblocks : 1)), '_'); + s += i->blocks.empty() ? "." : i->start_block_height == blockchain_height ? "m" : "o"; + expected = i->start_block_height + i->nblocks; + } + ++i; + } s += "]"; return s; } @@ -183,16 +222,31 @@ bool block_queue::requested(const crypto::hash &hash) const return requested_internal(hash); } -std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) +bool block_queue::have(const crypto::hash &hash) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + return have_blocks.find(hash) != have_blocks.end(); +} + +std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) { boost::unique_lock<boost::recursive_mutex> lock(mutex); + MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height + << ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " << + blockchain_height << ", block hashes size " << block_hashes.size()); if (last_block_height < first_block_height || max_blocks == 0) { MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks); return std::make_pair(0, 0); } + if (block_hashes.size() > last_block_height) + { + MDEBUG("reserve_span: more block hashes than fit within last_block_height: " << block_hashes.size() << " and " << last_block_height); + return std::make_pair(0, 0); + } + // skip everything we've already requested uint64_t span_start_height = last_block_height - block_hashes.size() + 1; std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); while (i != block_hashes.end() && requested_internal(*i)) @@ -200,55 +254,57 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei ++i; ++span_start_height; } + + // if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there + const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed); + MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed " + << epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE); + if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE) + { + MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height << + "(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed); + span_start_height = next_unpruned_height; + } + MDEBUG("span_start_height: " <<span_start_height); + const uint64_t block_hashes_start_height = last_block_height - block_hashes.size() + 1; + if (span_start_height >= block_hashes.size() + block_hashes_start_height) + { + MDEBUG("Out of hashes, cannot reserve"); + return std::make_pair(0, 0); + } + + i = block_hashes.begin() + span_start_height - block_hashes_start_height; + while (i != block_hashes.end() && requested_internal(*i)) + { + ++i; + ++span_start_height; + } + uint64_t span_length = 0; std::vector<crypto::hash> hashes; - while (i != block_hashes.end() && span_length < max_blocks) + while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed)) { hashes.push_back(*i); ++i; ++span_length; } if (span_length == 0) + { + MDEBUG("span_length 0, cannot reserve"); return std::make_pair(0, 0); + } MDEBUG("Reserving span " << span_start_height << " - " << (span_start_height + span_length - 1) << " for " << connection_id); add_blocks(span_start_height, span_length, connection_id, time); set_span_hashes(span_start_height, connection_id, hashes); return std::make_pair(span_start_height, span_length); } -bool block_queue::is_blockchain_placeholder(const span &span) const -{ - return span.connection_id == boost::uuids::nil_uuid(); -} - -std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const -{ - boost::unique_lock<boost::recursive_mutex> lock(mutex); - if (blocks.empty()) - return std::make_pair(0, 0); - block_map::const_iterator i = blocks.begin(); - if (!is_blockchain_placeholder(*i)) - return std::make_pair(0, 0); - uint64_t current_height = i->start_block_height + i->nblocks - 1; - ++i; - if (i == blocks.end()) - return std::make_pair(0, 0); - uint64_t first_span_height = i->start_block_height; - if (first_span_height <= current_height + 1) - return std::make_pair(0, 0); - MDEBUG("Found gap at start of spans: last blockchain block height " << current_height << ", first span's block height " << first_span_height); - print(); - return std::make_pair(current_height + 1, first_span_height - current_height - 1); -} - std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return std::make_pair(0, 0); block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return std::make_pair(0, 0); if (!i->blocks.empty()) @@ -259,6 +315,16 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vecto return std::make_pair(i->start_block_height, i->nblocks); } +void block_queue::reset_next_span_time(boost::posix_time::ptime t) +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + CHECK_AND_ASSERT_THROW_MES(!blocks.empty(), "No next span to reset time"); + block_map::iterator i = blocks.begin(); + CHECK_AND_ASSERT_THROW_MES(i != blocks.end(), "No next span to reset time"); + CHECK_AND_ASSERT_THROW_MES(i->blocks.empty(), "Next span is not empty"); + (boost::posix_time::ptime&)i->time = t; // sod off, time doesn't influence sorting +} + void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -283,8 +349,6 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_ if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; for (; i != blocks.end(); ++i) { if (!filled || !i->blocks.empty()) @@ -298,19 +362,34 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_ return false; } -bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const +bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return false; if (i->connection_id != connection_id) return false; filled = !i->blocks.empty(); + time = i->time; + return true; +} + +bool block_queue::has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return false; + block_map::const_iterator i = blocks.begin(); + if (i == blocks.end()) + return false; + if (i->start_block_height > height) + return false; + filled = !i->blocks.empty(); + time = i->time; + connection_id = i->connection_id; return true; } @@ -330,8 +409,6 @@ size_t block_queue::get_num_filled_spans_prefix() const if (blocks.empty()) return 0; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; size_t size = 0; while (i != blocks.end() && !i->blocks.empty()) { @@ -416,12 +493,35 @@ float block_queue::get_speed(const boost::uuids::uuid &connection_id) const return speed; } -bool block_queue::foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder) const +float block_queue::get_download_rate(const boost::uuids::uuid &connection_id) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + float conn_rate = -1.f; + for (const auto &span: blocks) + { + if (span.blocks.empty()) + continue; + if (span.connection_id != connection_id) + continue; + // note that the average below does not average over the whole set, but over the + // previous pseudo average and the latest rate: this gives much more importance + // to the latest measurements, which is fine here + if (conn_rate < 0.f) + conn_rate = span.rate; + else + conn_rate = (conn_rate + span.rate) / 2; + } + + if (conn_rate < 0) + conn_rate = 0.0f; + MTRACE("Download rate for " << connection_id << ": " << conn_rate << " b/s"); + return conn_rate; +} + +bool block_queue::foreach(std::function<bool(const span&)> f) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); block_map::const_iterator i = blocks.begin(); - if (!include_blockchain_placeholder && i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) if (!f(*i++)) return false; diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 9cce95075..1bef01d67 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -76,22 +76,26 @@ namespace cryptonote void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; - std::string get_overview() const; - std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); - bool is_blockchain_placeholder(const span &span) const; - std::pair<uint64_t, uint64_t> get_start_gap_span() const; + std::string get_overview(uint64_t blockchain_height) const; + bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const; + std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + uint64_t get_next_needed_height(uint64_t blockchain_height) const; std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time()); void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes); bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; - bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; + bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const; + bool has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; size_t get_num_filled_spans() const; crypto::hash get_last_known_hash(const boost::uuids::uuid &connection_id) const; bool has_spans(const boost::uuids::uuid &connection_id) const; float get_speed(const boost::uuids::uuid &connection_id) const; - bool foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder = false) const; + float get_download_rate(const boost::uuids::uuid &connection_id) const; + bool foreach(std::function<bool(const span&)> f) const; bool requested(const crypto::hash &hash) const; + bool have(const crypto::hash &hash) const; private: void erase_block(block_map::iterator j); @@ -101,5 +105,6 @@ namespace cryptonote block_map blocks; mutable boost::recursive_mutex mutex; std::unordered_set<crypto::hash> requested_hashes; + std::unordered_set<crypto::hash> have_blocks; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index d5bb50930..d582e3e9c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -48,11 +48,13 @@ namespace cryptonote bool incoming; bool localhost; bool local_ip; + bool ssl; std::string address; std::string host; std::string ip; std::string port; + uint16_t rpc_port; std::string peer_id; @@ -78,6 +80,8 @@ namespace cryptonote uint64_t height; + uint32_t pruning_seed; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(incoming) KV_SERIALIZE(localhost) @@ -86,6 +90,7 @@ namespace cryptonote KV_SERIALIZE(host) KV_SERIALIZE(ip) KV_SERIALIZE(port) + KV_SERIALIZE(rpc_port) KV_SERIALIZE(peer_id) KV_SERIALIZE(recv_count) KV_SERIALIZE(recv_idle_time) @@ -100,6 +105,7 @@ namespace cryptonote KV_SERIALIZE(support_flags) KV_SERIALIZE(connection_id) KV_SERIALIZE(height) + KV_SERIALIZE(pruning_seed) END_KV_SERIALIZE_MAP() }; @@ -124,7 +130,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 1; - struct request + struct request_t { block_complete_entry b; uint64_t current_blockchain_height; @@ -134,6 +140,7 @@ namespace cryptonote KV_SERIALIZE(current_blockchain_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; /************************************************************************/ @@ -143,7 +150,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 2; - struct request + struct request_t { std::vector<blobdata> txs; std::string _; // padding @@ -153,6 +160,7 @@ namespace cryptonote KV_SERIALIZE(_) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; /************************************************************************/ /* */ @@ -161,7 +169,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 3; - struct request + struct request_t { std::vector<crypto::hash> txs; std::vector<crypto::hash> blocks; @@ -171,13 +179,14 @@ namespace cryptonote KV_SERIALIZE_CONTAINER_POD_AS_BLOB(blocks) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; struct NOTIFY_RESPONSE_GET_OBJECTS { const static int ID = BC_COMMANDS_POOL_BASE + 4; - struct request + struct request_t { std::vector<blobdata> txs; std::vector<block_complete_entry> blocks; @@ -191,6 +200,7 @@ namespace cryptonote KV_SERIALIZE(current_blockchain_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; @@ -200,12 +210,14 @@ namespace cryptonote uint64_t cumulative_difficulty; crypto::hash top_id; uint8_t top_version; + uint32_t pruning_seed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(current_height) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE_VAL_POD_AS_BLOB(top_id) KV_SERIALIZE_OPT(top_version, (uint8_t)0) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; @@ -213,7 +225,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 6; - struct request + struct request_t { std::list<crypto::hash> block_ids; /*IDs of the first 10 blocks are sequential, next goes with pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ @@ -221,13 +233,14 @@ namespace cryptonote KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; struct NOTIFY_RESPONSE_CHAIN_ENTRY { const static int ID = BC_COMMANDS_POOL_BASE + 7; - struct request + struct request_t { uint64_t start_height; uint64_t total_height; @@ -241,6 +254,7 @@ namespace cryptonote KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; /************************************************************************/ @@ -250,7 +264,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 8; - struct request + struct request_t { block_complete_entry b; uint64_t current_blockchain_height; @@ -260,6 +274,7 @@ namespace cryptonote KV_SERIALIZE(current_blockchain_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; /************************************************************************/ @@ -269,7 +284,7 @@ namespace cryptonote { const static int ID = BC_COMMANDS_POOL_BASE + 9; - struct request + struct request_t { crypto::hash block_hash; uint64_t current_blockchain_height; @@ -281,6 +296,7 @@ namespace cryptonote KV_SERIALIZE_CONTAINER_POD_AS_BLOB(missing_tx_indices) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index 6d9ad9028..225418980 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -2,7 +2,7 @@ /// @author rfree (current maintainer in monero.cc project) /// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 618b635cc..0927b5d7f 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -2,7 +2,7 @@ /// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) /// @brief This is the original cryptonote protocol network-events handler, modified by us -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,6 +43,7 @@ #include "cryptonote_protocol_defs.h" #include "cryptonote_protocol_handler_common.h" #include "block_queue.h" +#include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" #include "cryptonote_basic/cryptonote_stat_info.h" #include <boost/circular_buffer.hpp> @@ -109,6 +110,11 @@ namespace cryptonote const block_queue &get_block_queue() const { return m_block_queue; } void stop(); void on_connection_close(cryptonote_connection_context &context); + void set_max_out_peers(unsigned int max) { m_max_out_peers = max; } + void set_no_sync(bool value) { m_no_sync = value; } + std::string get_peers_overview() const; + std::pair<uint32_t, uint32_t> get_next_needed_pruning_stripe() const; + bool needs_new_sync_connections() const; private: //----------------- commands handlers ---------------------------------------------- int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context); @@ -125,14 +131,18 @@ namespace cryptonote virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context); //---------------------------------------------------------------------------------- //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context); + bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe); bool request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span = false); size_t get_synchronizing_connections_count(); bool on_connection_synchronized(); - bool should_download_next_span(cryptonote_connection_context& context) const; + bool should_download_next_span(cryptonote_connection_context& context, bool standby); void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); bool check_standby_peers(); + bool update_sync_search(); int try_add_next_blocks(cryptonote_connection_context &context); + void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe); + void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const; t_core& m_core; @@ -141,10 +151,18 @@ namespace cryptonote std::atomic<uint32_t> m_syncronized_connections_count; std::atomic<bool> m_synchronized; std::atomic<bool> m_stopping; + std::atomic<bool> m_no_sync; boost::mutex m_sync_lock; block_queue m_block_queue; epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker; epee::math_helper::once_a_time_milliseconds<100> m_standby_checker; + epee::math_helper::once_a_time_seconds<101> m_sync_search_checker; + std::atomic<unsigned int> m_max_out_peers; + tools::PerformanceTimer m_sync_timer, m_add_timer; + uint64_t m_last_add_end_time; + uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded; + uint64_t m_sync_download_chain_size, m_sync_download_objects_size; + size_t m_block_download_max_size; boost::mutex m_buffer_mutex; double get_avg_block_size(); @@ -157,16 +175,7 @@ namespace cryptonote std::string blob; epee::serialization::store_t_to_binary(arg, blob); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, blob, context); - } - - template<class t_parameter> - bool relay_post_notify(typename t_parameter::request& arg, cryptonote_connection_context& exclude_context) - { - LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(exclude_context) << "] post relay " << typeid(t_parameter).name() << " -->"); - std::string arg_buff; - epee::serialization::store_t_to_binary(arg, arg_buff); - return m_p2p->relay_notify_to_all(t_parameter::ID, arg_buff, exclude_context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::strspan<uint8_t>(blob), context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 999ec5650..b33867e8b 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -2,7 +2,7 @@ /// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) /// @brief This is the original cryptonote protocol network-events handler, modified by us -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -42,17 +42,24 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "profile_tools.h" #include "net/network_throttle-detail.hpp" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.cn" #define MLOG_P2P_MESSAGE(x) MCINFO("net.p2p.msg", context << x) +#define MLOG_PEER_STATE(x) \ + MCINFO(MONERO_DEFAULT_LOG_CATEGORY, context << "[" << epee::string_tools::to_string_hex(context.m_pruning_seed) << "] state: " << x << " in state " << cryptonote::get_protocol_state_string(context.m_state)) -#define BLOCK_QUEUE_NBLOCKS_THRESHOLD 10 // chunks of N blocks +#define BLOCK_QUEUE_NSPANS_THRESHOLD 10 // chunks of N blocks #define BLOCK_QUEUE_SIZE_THRESHOLD (100*1024*1024) // MB -#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (5 * 1000000) // microseconds +#define BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS 1000 +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY (5 * 1000000) // microseconds +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (30 * 1000000) // microseconds #define IDLE_PEER_KICK_TIME (600 * 1000000) // microseconds #define PASSIVE_PEER_KICK_TIME (60 * 1000000) // microseconds +#define DROP_ON_SYNC_WEDGE_THRESHOLD (30 * 1000000000ull) // nanoseconds +#define LAST_ACTIVITY_STALL_THRESHOLD (2.0f) // seconds namespace cryptonote { @@ -65,7 +72,8 @@ namespace cryptonote m_p2p(p_net_layout), m_syncronized_connections_count(0), m_synchronized(offline), - m_stopping(false) + m_stopping(false), + m_no_sync(false) { if(!m_p2p) @@ -75,6 +83,19 @@ namespace cryptonote template<class t_core> bool t_cryptonote_protocol_handler<t_core>::init(const boost::program_options::variables_map& vm) { + m_sync_timer.pause(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + + m_block_download_max_size = command_line::get_arg(vm, cryptonote::arg_block_download_max_size); + return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -103,9 +124,12 @@ namespace cryptonote if(context.m_state == cryptonote_connection_context::state_synchronizing) { NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); + context.m_needed_objects.clear(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } else if(context.m_state == cryptonote_connection_context::state_standby) { @@ -203,11 +227,12 @@ namespace cryptonote cnx.host = cntxt.m_remote_address.host_str(); cnx.ip = ""; cnx.port = ""; - if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) { cnx.ip = cnx.host; cnx.port = std::to_string(cntxt.m_remote_address.as<epee::net_utils::ipv4_network_address>().port()); } + cnx.rpc_port = cntxt.m_rpc_port; std::stringstream peer_id_str; peer_id_str << std::hex << std::setw(16) << peer_id; @@ -245,8 +270,10 @@ namespace cryptonote cnx.current_upload = cntxt.m_current_speed_up / 1024; cnx.connection_id = epee::string_tools::pod_to_hex(cntxt.m_connection_id); + cnx.ssl = cntxt.m_ssl; cnx.height = cntxt.m_remote_blockchain_height; + cnx.pruning_seed = cntxt.m_pruning_seed; connections.push_back(cnx); @@ -279,7 +306,23 @@ namespace cryptonote } } + // reject weird pruning schemes + if (hshd.pruning_seed) + { + const uint32_t log_stripes = tools::get_pruning_log_stripes(hshd.pruning_seed); + if (log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES || tools::get_pruning_stripe(hshd.pruning_seed) > (1u << log_stripes)) + { + MWARNING(context << " peer claim unexpected pruning seed " << epee::string_tools::to_string_hex(hshd.pruning_seed) << ", disconnecting"); + return false; + } + } + context.m_remote_blockchain_height = hshd.current_height; + context.m_pruning_seed = hshd.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + context.m_pruning_seed = tools::make_pruning_seed(1 + (context.m_remote_address.as<epee::net_utils::ipv4_network_address>().ip()) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); + LOG_INFO_CC(context, "New connection posing as pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ", seed address " << &context.m_pruning_seed); +#endif uint64_t target = m_core.get_target_blockchain_height(); if (target == 0) @@ -293,12 +336,18 @@ namespace cryptonote return true; } + // No chain synchronization over hidden networks (tor, i2p, etc.) + if(context.m_remote_address.get_zone() != epee::net_utils::zone::public_) + { + context.m_state = cryptonote_connection_context::state_normal; + return true; + } + if (hshd.current_height > target) { /* As I don't know if accessing hshd from core could be a good practice, I prefer pushing target height to the core at the same time it is pushed to the user. Nz. */ - m_core.set_target_blockchain_height((hshd.current_height)); int64_t diff = static_cast<int64_t>(hshd.current_height) - static_cast<int64_t>(m_core.get_current_blockchain_height()); uint64_t abs_diff = std::abs(diff); uint64_t max_block_height = std::max(hshd.current_height,m_core.get_current_blockchain_height()); @@ -309,14 +358,38 @@ namespace cryptonote << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); if (hshd.current_height >= m_core.get_current_blockchain_height() + 5) // don't switch to unsafe mode just for a few blocks + { m_core.safesyncmode(false); + } + if (m_core.get_target_blockchain_height() == 0) // only when sync starts + { + m_sync_timer.resume(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + } + m_core.set_target_blockchain_height((hshd.current_height)); } - LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); + MINFO(context << "Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); + + if (m_no_sync) + { + context.m_state = cryptonote_connection_context::state_normal; + return true; + } + context.m_state = cryptonote_connection_context::state_synchronizing; //let the socket to send response to handshake, but request callback, to let send request data after response LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -327,6 +400,7 @@ namespace cryptonote hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); hshd.current_height +=1; + hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -353,7 +427,14 @@ namespace cryptonote m_core.pause_mine(); std::vector<block_complete_entry> blocks; blocks.push_back(arg.b); - m_core.prepare_handle_incoming_blocks(blocks); + std::vector<block> pblocks; + if (!m_core.prepare_handle_incoming_blocks(blocks, pblocks)) + { + LOG_PRINT_CCONTEXT_L1("Block verification failed: prepare_handle_incoming_blocks failed, dropping connection"); + drop_connection(context, false, false); + m_core.resume_mine(); + return 1; + } for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -369,7 +450,7 @@ namespace cryptonote } block_verification_context bvc = boost::value_initialized<block_verification_context>(); - m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block + m_core.handle_incoming_block(arg.b.block, pblocks.empty() ? NULL : &pblocks[0], bvc); // got block from handle_notify_new_block if (!m_core.cleanup_handle_incoming_blocks(true)) { LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks"); @@ -389,11 +470,14 @@ namespace cryptonote relay_block(arg, context); }else if(bvc.m_marked_as_orphaned) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } return 1; @@ -616,6 +700,7 @@ namespace cryptonote missing_tx_req.missing_tx_indices = std::move(need_tx_indices); m_core.resume_mine(); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_FLUFFY_MISSING_TX: missing_tx_indices.size()=" << missing_tx_req.missing_tx_indices.size() ); post_notify<NOTIFY_REQUEST_FLUFFY_MISSING_TX>(missing_tx_req, context); } else // whoo-hoo we've got em all .. @@ -628,10 +713,16 @@ namespace cryptonote std::vector<block_complete_entry> blocks; blocks.push_back(b); - m_core.prepare_handle_incoming_blocks(blocks); + std::vector<block> pblocks; + if (!m_core.prepare_handle_incoming_blocks(blocks, pblocks)) + { + LOG_PRINT_CCONTEXT_L0("Failure in prepare_handle_incoming_blocks"); + m_core.resume_mine(); + return 1; + } block_verification_context bvc = boost::value_initialized<block_verification_context>(); - m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block + m_core.handle_incoming_block(arg.b.block, pblocks.empty() ? NULL : &pblocks[0], bvc); // got block from handle_notify_new_block if (!m_core.cleanup_handle_incoming_blocks(true)) { LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks"); @@ -656,11 +747,14 @@ namespace cryptonote } else if( bvc.m_marked_as_orphaned ) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } } } @@ -747,7 +841,7 @@ namespace cryptonote fluffy_response.b.txs.push_back(t_serializable_object_to_blob(tx)); } - LOG_PRINT_CCONTEXT_L2 + MLOG_P2P_MESSAGE ( "-->>NOTIFY_RESPONSE_FLUFFY_MISSING_TX: " << ", txs.size()=" << fluffy_response.b.txs.size() @@ -811,7 +905,7 @@ namespace cryptonote drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify<NOTIFY_RESPONSE_GET_OBJECTS>(rsp, context); //handler_response_blocks_now(sizeof(rsp)); // XXX @@ -834,11 +928,14 @@ namespace cryptonote return avg / m_avg_buffer.size(); } - template<class t_core> int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); + MLOG_PEER_STATE("received objects"); + + boost::posix_time::ptime request_time = context.m_last_request_time; + context.m_last_request_time = boost::date_time::not_a_date_time; // calculate size of request size_t size = 0; @@ -860,6 +957,8 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_buffer_mutex); m_avg_buffer.push_back(size); } + ++m_sync_spans_downloaded; + m_sync_download_objects_size += size; MDEBUG(context << " downloaded " << size << " bytes worth of blocks"); /*using namespace boost::chrono; @@ -874,6 +973,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height << " < m_last_response_height=" << context.m_last_response_height << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -898,6 +998,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong block: failed to parse and validate block: " << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (b.miner_tx.vin.size() != 1 || b.miner_tx.vin.front().type() != typeid(txin_gen)) @@ -905,6 +1006,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong block: block: miner tx does not have exactly one txin_gen input" << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (start_height == std::numeric_limits<uint64_t>::max()) @@ -917,6 +1019,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << " wasn't requested, dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if(b.tx_hashes.size() != block_entry.txs.size()) @@ -924,6 +1027,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << ", tx_hashes.size()=" << b.tx_hashes.size() << " mismatch with block_complete_entry.m_txs.size()=" << block_entry.txs.size() << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -931,34 +1035,27 @@ namespace cryptonote block_hashes.push_back(block_hash); } - if(context.m_requested_objects.size()) + if(!context.m_requested_objects.empty()) { - MERROR("returned not all requested objects (context.m_requested_objects.size()=" + MERROR(context << "returned not all requested objects (context.m_requested_objects.size()=" << context.m_requested_objects.size() << "), dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } - // get the last parsed block, which should be the highest one - const crypto::hash last_block_hash = cryptonote::get_block_hash(b); - if(m_core.have_block(last_block_hash)) - { - const uint64_t subchain_height = start_height + arg.blocks.size(); - LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); - m_block_queue.remove_spans(context.m_connection_id, start_height); - goto skip; - } - { MLOG_YELLOW(el::Level::Debug, context << " Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() - << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1)); + << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1) << + " (pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); // add that new span to the block queue - const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const boost::posix_time::time_duration dt = now - request_time; const float rate = size * 1e6 / (dt.total_microseconds() + 1); - MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1e3) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); + MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1024) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); m_block_queue.add_blocks(start_height, arg.blocks, context.m_connection_id, rate, blocks_size); + const crypto::hash last_block_hash = cryptonote::get_block_hash(b); context.m_last_known_hash = last_block_hash; if (!m_core.get_test_drop_download() || !m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing @@ -966,7 +1063,6 @@ namespace cryptonote } } -skip: try_add_next_blocks(context); return 1; } @@ -983,15 +1079,22 @@ skip: const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; if (!sync.owns_lock()) { - MINFO("Failed to lock m_sync_lock, going back to download"); + MINFO(context << "Failed to lock m_sync_lock, going back to download"); goto skip; } MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); + MLOG_PEER_STATE("adding blocks"); { m_core.pause_mine(); - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( - boost::bind(&t_core::resume_mine, &m_core)); + m_add_timer.resume(); + bool starting = true; + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([this, &starting]() { + m_add_timer.pause(); + m_core.resume_mine(); + if (!starting) + m_last_add_end_time = tools::get_tick_count(); + }); while (1) { @@ -1004,22 +1107,38 @@ skip: MDEBUG(context << " no next span found, going back to download"); break; } - MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) - << ", we need " << previous_height); if (blocks.empty()) { - MERROR("Next span has no blocks"); + MERROR(context << "Next span has no blocks"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } + MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) + << ", we need " << previous_height); + block new_block; + if (!parse_and_validate_block_from_blob(blocks.back().block, new_block)) + { + MERROR(context << "Failed to parse block, but it should already have been parsed"); + m_block_queue.remove_spans(span_connection_id, start_height); + continue; + } + const crypto::hash last_block_hash = cryptonote::get_block_hash(new_block); + if (m_core.have_block(last_block_hash)) + { + const uint64_t subchain_height = start_height + blocks.size(); + LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); + m_block_queue.remove_spans(span_connection_id, start_height); + ++m_sync_old_spans_downloaded; + continue; + } if (!parse_and_validate_block_from_blob(blocks.front().block, new_block)) { - MERROR("Failed to parse block, but it should already have been parsed"); + MERROR(context << "Failed to parse block, but it should already have been parsed"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } bool parent_known = m_core.have_block(new_block.prev_id); if (!parent_known) @@ -1032,6 +1151,24 @@ skip: bool parent_requested = m_block_queue.requested(new_block.prev_id); if (!parent_requested) { + // we might be able to ask for that block directly, as we now can request out of order, + // otherwise we continue out of order, unless this block is the one we need, in which + // case we request block hashes, though it might be safer to disconnect ? + if (start_height > previous_height) + { + if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - dropping connection"); + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, true); + return 1; + } + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - back to download"); + + goto skip; + } + // this can happen if a connection was sicced onto a late span, if it did not have those blocks, // since we don't know that at the sic time LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - querying block hashes"); @@ -1047,12 +1184,33 @@ skip: } const boost::posix_time::ptime start = boost::posix_time::microsec_clock::universal_time(); - context.m_last_request_time = start; - m_core.prepare_handle_incoming_blocks(blocks); + if (starting) + { + starting = false; + if (m_last_add_end_time) + { + const uint64_t tnow = tools::get_tick_count(); + const uint64_t ns = tools::ticks_to_ns(tnow - m_last_add_end_time); + MINFO("Restarting adding block after idle for " << ns/1e9 << " seconds"); + } + } + + std::vector<block> pblocks; + if (!m_core.prepare_handle_incoming_blocks(blocks, pblocks)) + { + LOG_ERROR_CCONTEXT("Failure in prepare_handle_incoming_blocks"); + return 1; + } + if (!pblocks.empty() && pblocks.size() != blocks.size()) + { + m_core.cleanup_handle_incoming_blocks(); + LOG_ERROR_CCONTEXT("Internal error: blocks.size() != block_entry.txs.size()"); + return 1; + } uint64_t block_process_time_full = 0, transactions_process_time_full = 0; - size_t num_txs = 0; + size_t num_txs = 0, blockidx = 0; for(const block_complete_entry& block_entry: blocks) { if (m_stopping) @@ -1104,7 +1262,7 @@ skip: TIME_MEASURE_START(block_process_time); block_verification_context bvc = boost::value_initialized<block_verification_context>(); - m_core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block + m_core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx], bvc, false); // <--- process block if(bvc.m_verifivation_failed) { @@ -1147,10 +1305,11 @@ skip: TIME_MEASURE_FINISH(block_process_time); block_process_time_full += block_process_time; + ++blockidx; } // each download block - MCINFO("sync-info", "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); + MDEBUG(context << "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); if (!m_core.cleanup_handle_incoming_blocks()) { @@ -1160,40 +1319,54 @@ skip: m_block_queue.remove_spans(span_connection_id, start_height); - if (m_core.get_current_blockchain_height() > previous_height) + const uint64_t current_blockchain_height = m_core.get_current_blockchain_height(); + if (current_blockchain_height > previous_height) { + const uint64_t target_blockchain_height = m_core.get_target_blockchain_height(); const boost::posix_time::time_duration dt = boost::posix_time::microsec_clock::universal_time() - start; + std::string progress_message = ""; + if (current_blockchain_height < target_blockchain_height) + { + uint64_t completion_percent = (current_blockchain_height * 100 / target_blockchain_height); + if (completion_percent == 100) // never show 100% if not actually up to date + completion_percent = 99; + progress_message = " (" + std::to_string(completion_percent) + "%, " + + std::to_string(target_blockchain_height - current_blockchain_height) + " left)"; + } + const uint32_t previous_stripe = tools::get_pruning_stripe(previous_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t current_stripe = tools::get_pruning_stripe(current_blockchain_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); std::string timing_message = ""; if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) timing_message = std::string(" (") + std::to_string(dt.total_microseconds()/1e6) + " sec, " - + std::to_string((m_core.get_current_blockchain_height() - previous_height) * 1e6 / dt.total_microseconds()) - + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued"; + + std::to_string((current_blockchain_height - previous_height) * 1e6 / dt.total_microseconds()) + + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued in " + + std::to_string(m_block_queue.get_num_filled_spans()) + " spans, stripe " + + std::to_string(previous_stripe) + " -> " + std::to_string(current_stripe); if (ELPP->vRegistry()->allowed(el::Level::Debug, "sync-info")) - timing_message += std::string(": ") + m_block_queue.get_overview(); - if(m_core.get_target_blockchain_height() == 0){ - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << timing_message); - } else { - const int completition_percent = (m_core.get_current_blockchain_height() * 100 / m_core.get_target_blockchain_height()); - if(completition_percent < 99) {//printing completion percent only if % is < of 99 cause for 99 >= this is useless - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << " (" << completition_percent << "% " << (m_core.get_target_blockchain_height() - m_core.get_current_blockchain_height()) - << " blocks remaining)" << timing_message); - } else { - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << timing_message); - } - } + timing_message += std::string(": ") + m_block_queue.get_overview(current_blockchain_height); + MGINFO_YELLOW("Synced " << current_blockchain_height << "/" << target_blockchain_height + << progress_message << timing_message); + if (previous_stripe != current_stripe) + notify_new_stripe(context, current_stripe); } } } - if (should_download_next_span(context)) + MLOG_PEER_STATE("stopping adding blocks"); + + if (should_download_next_span(context, false)) { - context.m_needed_objects.clear(); - context.m_last_response_height = 0; force_next_span = true; } + else if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + if (!context.m_is_income) + { + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, false); + } + return 1; + } } skip: @@ -1207,10 +1380,33 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + void t_cryptonote_protocol_handler<t_core>::notify_new_stripe(cryptonote_connection_context& cntxt, uint32_t stripe) + { + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool + { + if (cntxt.m_connection_id == context.m_connection_id) + return true; + if (context.m_state == cryptonote_connection_context::state_normal) + { + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe && peer_stripe && peer_stripe != stripe) + return true; + context.m_state = cryptonote_connection_context::state_synchronizing; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); + } + return true; + }); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::on_idle() { m_idle_peer_kicker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::kick_idle_peers, this)); m_standby_checker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::check_standby_peers, this)); + m_sync_search_checker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::update_sync_search, this)); return m_core.on_idle(); } //------------------------------------------------------------------------------------------------------------------------ @@ -1218,30 +1414,65 @@ skip: bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector<boost::uuids::uuid> kick_connections; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { - if (context.m_state == cryptonote_connection_context::state_synchronizing || context.m_state == cryptonote_connection_context::state_before_handshake) + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) { - const bool passive = context.m_state == cryptonote_connection_context::state_before_handshake; const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); const boost::posix_time::time_duration dt = now - context.m_last_request_time; - const int64_t threshold = passive ? PASSIVE_PEER_KICK_TIME : IDLE_PEER_KICK_TIME; - if (dt.total_microseconds() > threshold) + if (dt.total_microseconds() > IDLE_PEER_KICK_TIME) { - MINFO(context << " kicking " << (passive ? "passive" : "idle") << " peer"); - kick_connections.push_back(context.m_connection_id); + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago"); + LOG_PRINT_CCONTEXT_L2("requesting callback"); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + ++context.m_callback_request_count; + m_p2p->request_callback(context); } } return true; }); - for (const boost::uuids::uuid &conn_id: kick_connections) + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::update_sync_search() + { + const uint64_t target = m_core.get_target_blockchain_height(); + const uint64_t height = m_core.get_current_blockchain_height(); + if (target > height) // if we're not synced yet, don't do it + return true; + + MTRACE("Checking for outgoing syncing peers..."); + unsigned n_syncing = 0, n_synced = 0; + boost::uuids::uuid last_synced_peer_id(boost::uuids::nil_uuid()); + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { - m_p2p->for_connection(conn_id, [this](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { - drop_connection(context, false, false); + if (!peer_id || context.m_is_income) // only consider connected outgoing peers return true; - }); + if (context.m_state == cryptonote_connection_context::state_synchronizing) + ++n_syncing; + if (context.m_state == cryptonote_connection_context::state_normal) + { + ++n_synced; + if (!context.m_anchor) + last_synced_peer_id = context.m_connection_id; + } + return true; + }); + MTRACE(n_syncing << " syncing, " << n_synced << " synced"); + + // if we're at max out peers, and not enough are syncing + if (n_synced + n_syncing >= m_max_out_peers && n_syncing < P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT && last_synced_peer_id != boost::uuids::nil_uuid()) + { + if (!m_p2p->for_connection(last_synced_peer_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + MINFO(ctx << "dropping synced peer, " << n_syncing << " syncing, " << n_synced << " synced"); + drop_connection(ctx, false, false); + return true; + })) + MDEBUG("Failed to find peer we wanted to drop"); } + return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1272,75 +1503,174 @@ skip: drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); post_notify<NOTIFY_RESPONSE_CHAIN_ENTRY>(r, context); return 1; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> - bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context) const + bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context, bool standby) { std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; + boost::uuids::uuid connection_id; std::pair<uint64_t, uint64_t> span; + bool filled; - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - MDEBUG(context << " we should download it as there is a gap"); - return true; - } - - // if the next span is not scheduled (or there is none) - span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); - if (span.second == 0) + const uint64_t blockchain_height = m_core.get_current_blockchain_height(); + if (context.m_remote_blockchain_height <= blockchain_height) + return false; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const bool has_next_block = tools::has_unpruned_block(blockchain_height, context.m_remote_blockchain_height, context.m_pruning_seed); + if (has_next_block) { - // we might be in a weird case where there is a filled next span, - // but it starts higher than the current height - uint64_t height; - std::vector<cryptonote::block_complete_entry> bcel; - if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) - return false; - if (height > m_core.get_current_blockchain_height()) + if (!m_block_queue.has_next_span(blockchain_height, filled, request_time, connection_id)) { - MDEBUG(context << " we should download it as the next block isn't scheduled"); + MDEBUG(context << " we should download it as no peer reserved it"); return true; } + if (!filled) + { + const long dt = (now - request_time).total_microseconds(); + if (dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + { + MDEBUG(context << " we should download it as it's not been received yet after " << dt/1e6); + return true; + } + + // in standby, be ready to double download early since we're idling anyway + // let the fastest peer trigger first + long threshold; + const double dl_speed = context.m_max_speed_down; + if (standby && dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY && dl_speed > 0) + { + bool download = false; + if (m_p2p->for_connection(connection_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + const time_t nowt = time(NULL); + const time_t time_since_last_recv = nowt - ctx.m_last_recv; + const float last_activity = std::min((float)time_since_last_recv, dt/1e6f); + const bool stalled = last_activity > LAST_ACTIVITY_STALL_THRESHOLD; + if (stalled) + { + MDEBUG(context << " we should download it as the downloading peer is stalling for " << nowt - ctx.m_last_recv << " seconds"); + download = true; + return true; + } + + // estimate the standby peer can give us 80% of its max speed + // and let it download if that speed is > N times as fast as the current one + // N starts at 10 after REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY, + // decreases to 1.25 at REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD, + // so that at times goes without the download being done, a retry becomes easier + const float max_multiplier = 10.f; + const float min_multiplier = 1.25f; + float multiplier = max_multiplier; + if (dt/1e6 >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) + { + multiplier = max_multiplier - (dt/1e6-REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) * (max_multiplier - min_multiplier) / (REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD - REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY); + multiplier = std::min(max_multiplier, std::max(min_multiplier, multiplier)); + } + if (dl_speed * .8f > ctx.m_current_speed_down * multiplier) + { + MDEBUG(context << " we should download it as we are substantially faster (" << dl_speed << " vs " + << ctx.m_current_speed_down << ", multiplier " << multiplier << " after " << dt/1e6 << " seconds)"); + download = true; + return true; + } + return true; + })) + { + if (download) + return true; + } + else + { + MWARNING(context << " we should download it as the downloading peer is unexpectedly not known to us"); + return true; + } + } + } + } + + return false; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe) + { + if (context.m_anchor) + { + MDEBUG(context << "This is an anchor peer, not dropping"); return false; } - // if it was scheduled by this particular peer - if (span_connection_id == context.m_connection_id) + if (context.m_pruning_seed == 0) + { + MDEBUG(context << "This peer is not striped, not dropping"); return false; + } - float span_speed = m_block_queue.get_speed(span_connection_id); - float speed = m_block_queue.get_speed(context.m_connection_id); - MDEBUG(context << " next span is scheduled for " << span_connection_id << ", speed " << span_speed << ", ours " << speed); - - // we try for that span too if: - // - we're substantially faster, or: - // - we're the fastest and the other one isn't (avoids a peer being waaaay slow but yet unmeasured) - // - the other one asked at least 5 seconds ago - if (span_speed < .25 && speed > .75f) + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe == peer_stripe) { - MDEBUG(context << " we should download it as we're substantially faster"); - return true; + MDEBUG(context << "This peer has needed stripe " << peer_stripe << ", not dropping"); + return false; } - if (speed == 1.0f && span_speed != 1.0f) + + if (!context.m_needed_objects.empty()) { - MDEBUG(context << " we should download it as we're the fastest peer"); - return true; + const uint64_t next_available_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + if (tools::has_unpruned_block(next_available_block_height, context.m_remote_blockchain_height, context.m_pruning_seed)) + { + MDEBUG(context << "This peer has unpruned next block at height " << next_available_block_height << ", not dropping"); + return false; + } } - const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); - if ((now - request_time).total_microseconds() > REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + + if (next_stripe > 0) { - MDEBUG(context << " we should download it as this span was requested long ago"); - return true; + unsigned int n_out_peers = 0, n_peers_on_next_stripe = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + if (ctx.m_state >= cryptonote_connection_context::state_synchronizing && tools::get_pruning_stripe(ctx.m_pruning_seed) == next_stripe) + ++n_peers_on_next_stripe; + return true; + }); + const uint32_t distance = (peer_stripe + (1<<CRYPTONOTE_PRUNING_LOG_STRIPES) - next_stripe) % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES); + if ((n_out_peers >= m_max_out_peers && n_peers_on_next_stripe == 0) || (distance > 1 && n_peers_on_next_stripe <= 2) || distance > 2) + { + MDEBUG(context << "we want seed " << next_stripe << ", and either " << n_out_peers << " is at max out peers (" + << m_max_out_peers << ") or distance " << distance << " from " << next_stripe << " to " << peer_stripe << + " is too large and we have only " << n_peers_on_next_stripe << " peers on next seed, dropping connection to make space"); + return true; + } } + MDEBUG(context << "End of checks, not dropping"); return false; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + void t_cryptonote_protocol_handler<t_core>::skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const + { + // take out blocks we already have + size_t skip = 0; + while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip]) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip])))) + { + // if we're popping the last hash, record it so we can ask again from that hash, + // this prevents never being able to progress on peers we get old hash lists from + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; + } + if (skip > 0) + { + MDEBUG(context << "skipping " << skip << "/" << context.m_needed_objects.size() << " blocks"); + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + } + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span) { // flush stale spans @@ -1357,47 +1687,102 @@ skip: { do { - size_t nblocks = m_block_queue.get_num_filled_spans(); + size_t nspans = m_block_queue.get_num_filled_spans(); size_t size = m_block_queue.get_data_size(); - if (nblocks < BLOCK_QUEUE_NBLOCKS_THRESHOLD || size < BLOCK_QUEUE_SIZE_THRESHOLD) + const uint64_t bc_height = m_core.get_current_blockchain_height(); + const auto next_needed_pruning_stripe = get_next_needed_pruning_stripe(); + const uint32_t add_stripe = tools::get_pruning_stripe(bc_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + const size_t block_queue_size_threshold = m_block_download_max_size ? m_block_download_max_size : BLOCK_QUEUE_SIZE_THRESHOLD; + bool queue_proceed = nspans < BLOCK_QUEUE_NSPANS_THRESHOLD || size < block_queue_size_threshold; + // get rid of blocks we already requested, or already have + skip_unneeded_hashes(context, true); + uint64_t next_needed_height = m_block_queue.get_next_needed_height(bc_height); + uint64_t next_block_height; + if (context.m_needed_objects.empty()) + next_block_height = next_needed_height; + else + next_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + bool stripe_proceed_main = (add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS); + bool stripe_proceed_secondary = tools::has_unpruned_block(next_block_height, context.m_remote_blockchain_height, context.m_pruning_seed); + bool proceed = stripe_proceed_main || (queue_proceed && stripe_proceed_secondary); + if (!stripe_proceed_main && !stripe_proceed_secondary && should_drop_connection(context, tools::get_pruning_stripe(next_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES))) + { + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections + } + + MDEBUG(context << "proceed " << proceed << " (queue " << queue_proceed << ", stripe " << stripe_proceed_main << "/" << + stripe_proceed_secondary << "), " << next_needed_pruning_stripe.first << "-" << next_needed_pruning_stripe.second << + " needed, bc add stripe " << add_stripe << ", we have " << peer_stripe << "), bc_height " << bc_height); + MDEBUG(context << " - next_block_height " << next_block_height << ", seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next_needed_height "<< next_needed_height); + MDEBUG(context << " - last_response_height " << context.m_last_response_height << ", m_needed_objects size " << context.m_needed_objects.size()); + + // if we're waiting for next span, try to get it before unblocking threads below, + // or a runaway downloading of future spans might happen + if (stripe_proceed_main && should_download_next_span(context, true)) + { + MDEBUG(context << " we should try for that next span too, we think we could get it faster, resuming"); + force_next_span = true; + MLOG_PEER_STATE("resuming"); + break; + } + + if (proceed) { if (context.m_state != cryptonote_connection_context::state_standby) { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", resuming"); + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", resuming"); + MLOG_PEER_STATE("resuming"); } break; } // this one triggers if all threads are in standby, which should not happen, // but happened at least once, so we unblock at least one thread if so - const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; if (sync.owns_lock()) { - LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); - break; - } + bool filled = false; + boost::posix_time::ptime time; + boost::uuids::uuid connection_id; + if (m_block_queue.has_next_span(m_core.get_current_blockchain_height(), filled, time, connection_id) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, and next span needed is ready, resuming"); + MLOG_PEER_STATE("resuming"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + else + { + sync.unlock(); - if (should_download_next_span(context)) - { - MDEBUG(context << " we should try for that next span too, we think we could get it faster, resuming"); - force_next_span = true; - break; + // if this has gone on for too long, drop incoming connection to guard against some wedge state + if (!context.m_is_income) + { + const uint64_t now = tools::get_tick_count(); + const uint64_t dt = now - m_last_add_end_time; + if (tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD) + { + MDEBUG(context << "Block addition seems to have wedged, dropping connection"); + return false; + } + } + } } if (context.m_state != cryptonote_connection_context::state_standby) { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", pausing"); + if (!queue_proceed) + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", pausing"); + else if (!stripe_proceed_main && !stripe_proceed_secondary) + LOG_DEBUG_CC(context, "We do not have the stripe required to download another block, pausing"); context.m_state = cryptonote_connection_context::state_standby; - } - - // this needs doing after we went to standby, so the callback knows what to do - bool filled; - if (m_block_queue.has_next_span(context.m_connection_id, filled) && !filled) - { - MDEBUG(context << " we have the next span, and it is scheduled, resuming"); - ++context.m_callback_request_count; - m_p2p->request_callback(context); - return true; + MLOG_PEER_STATE("pausing"); } return true; @@ -1405,7 +1790,9 @@ skip: context.m_state = cryptonote_connection_context::state_synchronizing; } - MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " << m_core.get_current_blockchain_height()); + MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span + << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " + << m_core.get_current_blockchain_height() << ", pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); if(context.m_needed_objects.size() || force_next_span) { //we know objects that we need, request this objects @@ -1414,28 +1801,6 @@ skip: size_t count = 0; const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height()); std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0); - { - MDEBUG(context << " checking for gap"); - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - const uint64_t first_block_height_known = context.m_last_response_height - context.m_needed_objects.size() + 1; - const uint64_t last_block_height_known = context.m_last_response_height; - const uint64_t first_block_height_needed = span.first; - const uint64_t last_block_height_needed = span.first + std::min(span.second, (uint64_t)count_limit) - 1; - MDEBUG(context << " gap found, span: " << span.first << " - " << span.first + span.second - 1 << " (" << last_block_height_needed << ")"); - MDEBUG(context << " current known hashes from " << first_block_height_known << " to " << last_block_height_known); - if (first_block_height_needed < first_block_height_known || last_block_height_needed > last_block_height_known) - { - MDEBUG(context << " we are missing some of the necessary hashes for this gap, requesting chain again"); - context.m_needed_objects.clear(); - context.m_last_response_height = 0; - start_from_current_chain = true; - goto skip; - } - MDEBUG(context << " we have the hashes for this gap"); - } - } if (force_next_span) { if (span.second == 0) @@ -1452,6 +1817,7 @@ skip: req.blocks.push_back(hash); context.m_requested_objects.insert(hash); } + m_block_queue.reset_next_span_time(); } } } @@ -1465,22 +1831,20 @@ skip: context.m_last_response_height = 0; goto skip; } - // take out blocks we already have - size_t skip = 0; - while (skip < context.m_needed_objects.size() && m_core.have_block(context.m_needed_objects[skip])) - { - // if we're popping the last hash, record it so we can ask again from that hash, - // this prevents never being able to progress on peers we get old hash lists from - if (skip + 1 == context.m_needed_objects.size()) - context.m_last_known_hash = context.m_needed_objects[skip]; - ++skip; - } - if (skip > 0) - context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + skip_unneeded_hashes(context, false); const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; - span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); + span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); + if (span.second > 0) + { + const uint32_t stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (context.m_pruning_seed && stripe != tools::get_pruning_stripe(context.m_pruning_seed)) + { + MDEBUG(context << " starting early on next seed (" << span.first << " with stripe " << stripe << + ", context seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); + } + } } if (span.second == 0 && !force_next_span) { @@ -1489,6 +1853,8 @@ skip: boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); + if (span.second > 0 && !tools::has_unpruned_block(span.first, context.m_remote_blockchain_height, context.m_pruning_seed)) + span = std::make_pair(0, 0); if (span.second > 0) { is_next = true; @@ -1534,17 +1900,67 @@ skip: } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front()); //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); + MLOG_PEER_STATE("requesting objects"); + return true; + } + + // if we're still around, we might be at a point where the peer is pruned, so we could either + // drop it to make space for other peers, or ask for a span further down the line + const uint32_t next_stripe = get_next_needed_pruning_stripe().first; + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe && peer_stripe && next_stripe != peer_stripe) + { + // at this point, we have to either close the connection, or start getting blocks past the + // current point, or become dormant + MDEBUG(context << "this peer is pruned at seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next stripe needed is " << next_stripe); + if (!context.m_is_income) + { + if (should_drop_connection(context, next_stripe)) + { + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections + } + } + // we'll get back stuck waiting for the go ahead + context.m_state = cryptonote_connection_context::state_normal; + MLOG_PEER_STATE("Nothing to do for now, switching to normal state"); return true; } } skip: context.m_needed_objects.clear(); + + // we might have been called from the "received chain entry" handler, and end up + // here because we can't use any of those blocks (maybe because all of them are + // actually already requested). In this case, if we can add blocks instead, do so + if (m_core.get_current_blockchain_height() < m_core.get_target_blockchain_height()) + { + const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + if (sync.owns_lock()) + { + uint64_t start_height; + std::vector<cryptonote::block_complete_entry> blocks; + boost::uuids::uuid span_connection_id; + bool filled = false; + if (m_block_queue.get_next_span(start_height, blocks, span_connection_id, filled) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); + MLOG_PEER_STATE("will try to add blocks next"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + } + } + if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry @@ -1567,8 +1983,9 @@ skip: //LOG_PRINT_CCONTEXT_L1("r = " << 200); context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); }else { CHECK_AND_ASSERT_MES(context.m_last_response_height == context.m_remote_blockchain_height-1 @@ -1608,9 +2025,25 @@ skip: << ENDL << "Use the \"help\" command to see the list of available commands." << ENDL << "**********************************************************************"); + m_sync_timer.pause(); + if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) + { + const uint64_t sync_time = m_sync_timer.value(); + const uint64_t add_time = m_add_timer.value(); + if (sync_time && add_time) + { + MCLOG_YELLOW(el::Level::Info, "sync-info", "Sync time: " << sync_time/1e9/60 << " min, idle time " << + (100.f * (1.0f - add_time / (float)sync_time)) << "%" << ", " << + (10 * m_sync_download_objects_size / 1024 / 1024) / 10.f << " + " << + (10 * m_sync_download_chain_size / 1024 / 1024) / 10.f << " MB downloaded, " << + 100.0f * m_sync_old_spans_downloaded / m_sync_spans_downloaded << "% old spans, " << + 100.0f * m_sync_bad_spans_downloaded / m_sync_spans_downloaded << "% bad spans"); + } + } m_core.on_synchronized(); } m_core.safesyncmode(true); + m_p2p->clear_used_stripe_peers(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1631,6 +2064,11 @@ skip: { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_CHAIN_ENTRY: m_block_ids.size()=" << arg.m_block_ids.size() << ", m_start_height=" << arg.start_height << ", m_total_height=" << arg.total_height); + MLOG_PEER_STATE("received chain"); + + context.m_last_request_time = boost::date_time::not_a_date_time; + + m_sync_download_chain_size += arg.m_block_ids.size() * sizeof(crypto::hash); if(!arg.m_block_ids.size()) { @@ -1644,7 +2082,14 @@ skip: drop_connection(context, true, false); return 1; } + MDEBUG(context << "first block hash " << arg.m_block_ids.front() << ", last " << arg.m_block_ids.back()); + if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() >= CRYPTONOTE_MAX_BLOCK_NUMBER) + { + LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_CHAIN_ENTRY, with total_height=" << arg.total_height << " and block_ids=" << arg.m_block_ids.size()); + drop_connection(context, false, false); + return 1; + } context.m_remote_blockchain_height = arg.total_height; context.m_last_response_height = arg.start_height + arg.m_block_ids.size()-1; if(context.m_last_response_height > context.m_remote_blockchain_height) @@ -1664,6 +2109,7 @@ skip: return 1; } + context.m_needed_objects.clear(); uint64_t added = 0; for(auto& bl_id: arg.m_block_ids) { @@ -1696,20 +2142,20 @@ skip: fluffy_arg.b.txs = fluffy_txs; // sort peers between fluffy ones and others - std::list<boost::uuids::uuid> fullConnections, fluffyConnections; + std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> fullConnections, fluffyConnections; m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { - if (peer_id && exclude_context.m_connection_id != context.m_connection_id) + if (peer_id && exclude_context.m_connection_id != context.m_connection_id && context.m_remote_address.get_zone() == epee::net_utils::zone::public_) { if(m_core.fluffy_blocks_enabled() && (support_flags & P2P_SUPPORT_FLAG_FLUFFY_BLOCKS)) { LOG_DEBUG_CC(context, "PEER SUPPORTS FLUFFY BLOCKS - RELAYING THIN/COMPACT WHATEVER BLOCK"); - fluffyConnections.push_back(context.m_connection_id); + fluffyConnections.push_back({context.m_remote_address.get_zone(), context.m_connection_id}); } else { LOG_DEBUG_CC(context, "PEER DOESN'T SUPPORT FLUFFY BLOCKS - RELAYING FULL BLOCK"); - fullConnections.push_back(context.m_connection_id); + fullConnections.push_back({context.m_remote_address.get_zone(), context.m_connection_id}); } } return true; @@ -1720,13 +2166,13 @@ skip: { std::string fluffyBlob; epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::strspan<uint8_t>(fluffyBlob), std::move(fluffyConnections)); } if (!fullConnections.empty()) { std::string fullBlob; epee::serialization::store_t_to_binary(arg, fullBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::strspan<uint8_t>(fullBlob), std::move(fullConnections)); } return true; @@ -1735,8 +2181,14 @@ skip: template<class t_core> bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context) { + const bool hide_tx_broadcast = + 1 < m_p2p->get_zone_count() && exclude_context.m_remote_address.get_zone() == epee::net_utils::zone::invalid; + + if (hide_tx_broadcast) + MDEBUG("Attempting to conceal origin of tx via anonymity network connection(s)"); + // no check for success, so tell core they're relayed unconditionally - const bool pad_transactions = m_core.pad_transactions(); + const bool pad_transactions = m_core.pad_transactions() || hide_tx_broadcast; size_t bytes = pad_transactions ? 9 /* header */ + 4 /* 1 + 'txs' */ + tools::get_varint_data(arg.txs.size()).size() : 0; for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end(); ++tx_blob_it) { @@ -1769,14 +2221,112 @@ skip: // if the size of _ moved enough, we might lose byte in size encoding, we don't care } - return relay_post_notify<NOTIFY_NEW_TRANSACTIONS>(arg, exclude_context); + std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections; + m_p2p->for_each_connection([hide_tx_broadcast, &exclude_context, &connections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) + { + const epee::net_utils::zone current_zone = context.m_remote_address.get_zone(); + const bool broadcast_to_peer = + peer_id && + (hide_tx_broadcast != bool(current_zone == epee::net_utils::zone::public_)) && + exclude_context.m_connection_id != context.m_connection_id; + + if (broadcast_to_peer) + connections.push_back({current_zone, context.m_connection_id}); + + return true; + }); + + if (connections.empty()) + MERROR("Transaction not relayed - no" << (hide_tx_broadcast ? " privacy": "") << " peers available"); + else + { + std::string fullBlob; + epee::serialization::store_t_to_binary(arg, fullBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<uint8_t>(fullBlob), std::move(connections)); + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + std::string t_cryptonote_protocol_handler<t_core>::get_peers_overview() const + { + std::stringstream ss; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + m_p2p->for_each_connection([&](const connection_context &ctx, nodetool::peerid_type peer_id, uint32_t support_flags) { + const uint32_t stripe = tools::get_pruning_stripe(ctx.m_pruning_seed); + char state_char = cryptonote::get_protocol_state_char(ctx.m_state); + ss << stripe + state_char; + if (ctx.m_last_request_time != boost::date_time::not_a_date_time) + ss << (((now - ctx.m_last_request_time).total_microseconds() > IDLE_PEER_KICK_TIME) ? "!" : "?"); + ss << + " "; + return true; + }); + return ss.str(); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + std::pair<uint32_t, uint32_t> t_cryptonote_protocol_handler<t_core>::get_next_needed_pruning_stripe() const + { + const uint64_t want_height_from_blockchain = m_core.get_current_blockchain_height(); + const uint64_t want_height_from_block_queue = m_block_queue.get_next_needed_height(want_height_from_blockchain); + const uint64_t want_height = std::max(want_height_from_blockchain, want_height_from_block_queue); + uint64_t blockchain_height = m_core.get_target_blockchain_height(); + // if we don't know the remote chain size yet, assume infinitely large so we get the right stripe if we're not near the tip + if (blockchain_height == 0) + blockchain_height = CRYPTONOTE_MAX_BLOCK_NUMBER; + const uint32_t next_pruning_stripe = tools::get_pruning_stripe(want_height, blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (next_pruning_stripe == 0) + return std::make_pair(0, 0); + // if we already have a few peers on this stripe, but none on next one, try next one + unsigned int n_next = 0, n_subsequent = 0, n_others = 0; + const uint32_t subsequent_pruning_stripe = 1 + next_pruning_stripe % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES); + m_p2p->for_each_connection([&](const connection_context &context, nodetool::peerid_type peer_id, uint32_t support_flags) { + if (context.m_state >= cryptonote_connection_context::state_synchronizing) + { + if (context.m_pruning_seed == 0 || tools::get_pruning_stripe(context.m_pruning_seed) == next_pruning_stripe) + ++n_next; + else if (tools::get_pruning_stripe(context.m_pruning_seed) == subsequent_pruning_stripe) + ++n_subsequent; + else + ++n_others; + } + return true; + }); + const bool use_next = (n_next > m_max_out_peers / 2 && n_subsequent <= 1) || (n_next > 2 && n_subsequent == 0); + const uint32_t ret_stripe = use_next ? subsequent_pruning_stripe: next_pruning_stripe; + MIDEBUG(const std::string po = get_peers_overview(), "get_next_needed_pruning_stripe: want height " << want_height << " (" << + want_height_from_blockchain << " from blockchain, " << want_height_from_block_queue << " from block queue), stripe " << + next_pruning_stripe << " (" << n_next << "/" << m_max_out_peers << " on it and " << n_subsequent << " on " << + subsequent_pruning_stripe << ", " << n_others << " others) -> " << ret_stripe << " (+" << + (ret_stripe - next_pruning_stripe + (1 << CRYPTONOTE_PRUNING_LOG_STRIPES)) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES) << + "), current peers " << po); + return std::make_pair(next_pruning_stripe, ret_stripe); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::needs_new_sync_connections() const + { + const uint64_t target = m_core.get_target_blockchain_height(); + const uint64_t height = m_core.get_current_blockchain_height(); + if (target && target <= height) + return false; + size_t n_out_peers = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + return true; + }); + if (n_out_peers >= m_max_out_peers) + return false; + return true; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> void t_cryptonote_protocol_handler<t_core>::drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans) { - LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << - ", add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); + LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << " (pruning seed " << + epee::string_tools::to_string_hex(context.m_pruning_seed) << + "), add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); if (add_fail) m_p2p->add_host_fail(context.m_remote_address); @@ -1803,6 +2353,7 @@ skip: } m_block_queue.flush_spans(context.m_connection_id, false); + MLOG_PEER_STATE("closed"); } //------------------------------------------------------------------------------------------------------------------------ diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 2b9f201ec..a67178c52 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 117790455..d9bfd9a20 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index cba71bf3b..32fdca5ea 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -96,6 +96,12 @@ namespace daemon_args , 0 }; + const command_line::arg_descriptor<bool> arg_public_node = { + "public-node" + , "Allow other users to use the node as a remote (restricted RPC mode, view-only commands) and advertise it over P2P" + , false + }; + const command_line::arg_descriptor<std::string> arg_zmq_rpc_bind_ip = { "zmq-rpc-bind-ip" , "IP for ZMQ RPC server to listen on" diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 853cde9c3..b324ab99d 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -48,9 +48,34 @@ t_command_parser_executor::t_command_parser_executor( bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& args) { - if (!args.empty()) return false; + if (args.size() > 3) + { + std::cout << "use: print_pl [white] [gray] [<limit>]" << std::endl; + return true; + } + + bool white = false; + bool gray = false; + size_t limit = 0; + for (size_t i = 0; i < args.size(); ++i) + { + if (args[i] == "white") + { + white = true; + } + else if (args[i] == "gray") + { + gray = true; + } + else if (!epee::string_tools::get_xtype_from_string(limit, args[i])) + { + std::cout << "unexpected argument: " << args[i] << std::endl; + return true; + } + } - return m_executor.print_peer_list(); + const bool print_both = !white && !gray; + return m_executor.print_peer_list(white | print_both, gray | print_both, limit); } bool t_command_parser_executor::print_peer_list_stats(const std::vector<std::string>& args) @@ -163,9 +188,21 @@ bool t_command_parser_executor::print_height(const std::vector<std::string>& arg bool t_command_parser_executor::print_block(const std::vector<std::string>& args) { + bool include_hex = false; + + // Assumes that optional flags come after mandatory argument <transaction_hash> + for (unsigned int i = 1; i < args.size(); ++i) { + if (args[i] == "+hex") + include_hex = true; + else + { + std::cout << "unexpected argument: " << args[i] << std::endl; + return true; + } + } if (args.empty()) { - std::cout << "expected: print_block (<block_hash> | <block_height>)" << std::endl; + std::cout << "expected: print_block (<block_hash> | <block_height>) [+hex]" << std::endl; return false; } @@ -173,14 +210,14 @@ bool t_command_parser_executor::print_block(const std::vector<std::string>& args try { uint64_t height = boost::lexical_cast<uint64_t>(arg); - return m_executor.print_block_by_height(height); + return m_executor.print_block_by_height(height, include_hex); } catch (const boost::bad_lexical_cast&) { crypto::hash block_hash; if (parse_hash256(arg, block_hash)) { - return m_executor.print_block_by_hash(block_hash); + return m_executor.print_block_by_hash(block_hash, include_hex); } } @@ -265,7 +302,7 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg { if(!args.size()) { - std::cout << "Please specify a wallet address to mine for: start_mining <addr> [<threads>]" << std::endl; + std::cout << "Please specify a wallet address to mine for: start_mining <addr> [<threads>|auto]" << std::endl; return true; } @@ -351,8 +388,15 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg if(args.size() >= 2) { - bool ok = epee::string_tools::get_xtype_from_string(threads_count, args[1]); - threads_count = (ok && 0 < threads_count) ? threads_count : 1; + if (args[1] == "auto" || args[1] == "autodetect") + { + threads_count = 0; + } + else + { + bool ok = epee::string_tools::get_xtype_from_string(threads_count, args[1]); + threads_count = (ok && 0 < threads_count) ? threads_count : 1; + } } m_executor.start_mining(info.address, threads_count, nettype, do_background_mining, ignore_battery); @@ -367,6 +411,11 @@ bool t_command_parser_executor::stop_mining(const std::vector<std::string>& args return m_executor.stop_mining(); } +bool t_command_parser_executor::mining_status(const std::vector<std::string>& args) +{ + return m_executor.mining_status(); +} + bool t_command_parser_executor::stop_daemon(const std::vector<std::string>& args) { if (!args.empty()) return false; @@ -705,4 +754,27 @@ bool t_command_parser_executor::version(const std::vector<std::string>& args) return true; } +bool t_command_parser_executor::prune_blockchain(const std::vector<std::string>& args) +{ + if (args.size() > 1) return false; + + if (args.empty() || args[0] != "confirm") + { + std::cout << "Warning: pruning from within monerod will not shrink the database file size." << std::endl; + std::cout << "Instead, parts of the file will be marked as free, so the file will not grow" << std::endl; + std::cout << "until that newly free space is used up. If you want a smaller file size now," << std::endl; + std::cout << "exit monerod and run monero-blockchain-prune (you will temporarily need more" << std::endl; + std::cout << "disk space for the database conversion though). If you are OK with the database" << std::endl; + std::cout << "file keeping the same size, re-run this command with the \"confirm\" parameter." << std::endl; + return true; + } + + return m_executor.prune_blockchain(); +} + +bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std::string>& args) +{ + return m_executor.check_blockchain_pruning(); +} + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index e2844e8b7..bec6e4522 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -6,7 +6,7 @@ */ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -97,6 +97,8 @@ public: bool stop_mining(const std::vector<std::string>& args); + bool mining_status(const std::vector<std::string>& args); + bool stop_daemon(const std::vector<std::string>& args); bool print_status(const std::vector<std::string>& args); @@ -142,6 +144,10 @@ public: bool pop_blocks(const std::vector<std::string>& args); bool version(const std::vector<std::string>& args); + + bool prune_blockchain(const std::vector<std::string>& args); + + bool check_blockchain_pruning(const std::vector<std::string>& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 527ed2cf1..94e4a8bf1 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -64,6 +64,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "print_pl" , std::bind(&t_command_parser_executor::print_peer_list, &m_parser, p::_1) + , "print_pl [white] [gray] [<limit>]" , "Print the current peer list." ); m_command_lookup.set_handler( @@ -103,8 +104,8 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "start_mining" , std::bind(&t_command_parser_executor::start_mining, &m_parser, p::_1) - , "start_mining <addr> [<threads>] [do_background_mining] [ignore_battery]" - , "Start mining for specified address. Defaults to 1 thread and no background mining." + , "start_mining <addr> [<threads>|auto] [do_background_mining] [ignore_battery]" + , "Start mining for specified address. Defaults to 1 thread and no background mining. Use \"auto\" to autodetect optimal number of threads." ); m_command_lookup.set_handler( "stop_mining" @@ -112,6 +113,11 @@ t_command_server::t_command_server( , "Stop mining." ); m_command_lookup.set_handler( + "mining_status" + , std::bind(&t_command_parser_executor::mining_status, &m_parser, p::_1) + , "Show current mining status." + ); + m_command_lookup.set_handler( "print_pool" , std::bind(&t_command_parser_executor::print_transaction_pool_long, &m_parser, p::_1) , "Print the transaction pool using a long format." @@ -292,6 +298,16 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::version, &m_parser, p::_1) , "Print version information." ); + m_command_lookup.set_handler( + "prune_blockchain" + , std::bind(&t_command_parser_executor::prune_blockchain, &m_parser, p::_1) + , "Prune the blockchain." + ); + m_command_lookup.set_handler( + "check_blockchain_pruning" + , std::bind(&t_command_parser_executor::check_blockchain_pruning, &m_parser, p::_1) + , "Check the blockchain pruning." + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h index aff74da45..c8e77f551 100644 --- a/src/daemon/command_server.h +++ b/src/daemon/command_server.h @@ -9,7 +9,7 @@ Passing RPC commands: */ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/core.h b/src/daemon/core.h index c15d8d236..91dbb7a4b 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 88fba8372..3d1d893ea 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -96,9 +96,11 @@ void t_daemon::init_options(boost::program_options::options_description & option } t_daemon::t_daemon( - boost::program_options::variables_map const & vm + boost::program_options::variables_map const & vm, + uint16_t public_rpc_port ) - : mp_internals{new t_internals{vm}} + : mp_internals{new t_internals{vm}}, + public_rpc_port(public_rpc_port) { zmq_rpc_bind_port = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_port); zmq_rpc_bind_address = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_ip); @@ -186,6 +188,12 @@ bool t_daemon::run(bool interactive) MINFO(std::string("ZMQ server started at ") + zmq_rpc_bind_address + ":" + zmq_rpc_bind_port + "."); + if (public_rpc_port > 0) + { + MGINFO("Public RPC port " << public_rpc_port << " will be advertised to other peers over P2P"); + mp_internals->p2p.get().set_rpc_port(public_rpc_port); + } + mp_internals->p2p.run(); // blocks until p2p goes down if (rpc_commands) diff --git a/src/daemon/daemon.h b/src/daemon/daemon.h index 1e356ef5f..d44173177 100644 --- a/src/daemon/daemon.h +++ b/src/daemon/daemon.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -43,11 +43,13 @@ private: void stop_p2p(); private: std::unique_ptr<t_internals> mp_internals; + uint16_t public_rpc_port; std::string zmq_rpc_bind_address; std::string zmq_rpc_bind_port; public: t_daemon( - boost::program_options::variables_map const & vm + boost::program_options::variables_map const & vm, + uint16_t public_rpc_port = 0 ); t_daemon(t_daemon && other); t_daemon & operator=(t_daemon && other); diff --git a/src/daemon/executor.cpp b/src/daemon/executor.cpp index fbc7d04fd..3719a253d 100644 --- a/src/daemon/executor.cpp +++ b/src/daemon/executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -59,21 +59,21 @@ namespace daemonize ) { LOG_PRINT_L0("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ") Daemonised"); - return t_daemon{vm}; + return t_daemon{vm, public_rpc_port}; } bool t_executor::run_non_interactive( boost::program_options::variables_map const & vm ) { - return t_daemon{vm}.run(false); + return t_daemon{vm, public_rpc_port}.run(false); } bool t_executor::run_interactive( boost::program_options::variables_map const & vm ) { - return t_daemon{vm}.run(true); + return t_daemon{vm, public_rpc_port}.run(true); } } diff --git a/src/daemon/executor.h b/src/daemon/executor.h index 79d70567a..61e4e1bbf 100644 --- a/src/daemon/executor.h +++ b/src/daemon/executor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -45,6 +45,10 @@ namespace daemonize static std::string const NAME; + t_executor(uint16_t public_rpc_port = 0) : public_rpc_port(public_rpc_port) + { + } + static void init_options( boost::program_options::options_description & configurable_options ); @@ -62,5 +66,8 @@ namespace daemonize bool run_interactive( boost::program_options::variables_map const & vm ); + + private: + uint16_t public_rpc_port; }; } diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 35017d9ef..c3ac24b70 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -39,6 +39,7 @@ #include "daemon/executor.h" #include "daemonizer/daemonizer.h" #include "misc_log_ex.h" +#include "net/parse.h" #include "p2p/net_node.h" #include "rpc/core_rpc_server.h" #include "rpc/rpc_args.h" @@ -56,6 +57,57 @@ namespace po = boost::program_options; namespace bf = boost::filesystem; +uint16_t parse_public_rpc_port(const po::variables_map &vm) +{ + const auto &public_node_arg = daemon_args::arg_public_node; + const bool public_node = command_line::get_arg(vm, public_node_arg); + if (!public_node) + { + return 0; + } + + std::string rpc_port_str; + const auto &restricted_rpc_port = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port; + if (!command_line::is_arg_defaulted(vm, restricted_rpc_port)) + { + rpc_port_str = command_line::get_arg(vm, restricted_rpc_port);; + } + else if (command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc)) + { + rpc_port_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + } + else + { + throw std::runtime_error("restricted RPC mode is required"); + } + + uint16_t rpc_port; + if (!string_tools::get_xtype_from_string(rpc_port, rpc_port_str)) + { + throw std::runtime_error("invalid RPC port " + rpc_port_str); + } + + const auto rpc_bind_address = command_line::get_arg(vm, cryptonote::rpc_args::descriptors().rpc_bind_ip); + const auto address = net::get_network_address(rpc_bind_address, rpc_port); + if (!address) { + throw std::runtime_error("failed to parse RPC bind address"); + } + if (address->get_zone() != epee::net_utils::zone::public_) + { + throw std::runtime_error(std::string(zone_to_string(address->get_zone())) + + " network zone is not supported, please check RPC server bind address"); + } + + if (address->is_loopback() || address->is_local()) + { + MLOG_RED(el::Level::Warning, "--" << public_node_arg.name + << " is enabled, but RPC server " << address->str() + << " may be unreachable from outside, please check RPC server bind address"); + } + + return rpc_port; +} + int main(int argc, char const * argv[]) { try { @@ -86,6 +138,7 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); + command_line::add_arg(core_settings, daemon_args::arg_public_node); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); @@ -288,7 +341,7 @@ int main(int argc, char const * argv[]) MINFO("Moving from main() into the daemonize now."); - return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm) ? 0 : 1; + return daemonizer::daemonize(argc, argv, daemonize::t_executor{parse_public_rpc_port(vm)}, vm) ? 0 : 1; } catch (std::exception const & ex) { diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h index 0f01c746d..267b99c89 100644 --- a/src/daemon/p2p.h +++ b/src/daemon/p2p.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/protocol.h b/src/daemon/protocol.h index fd1d1b638..51a2bce1f 100644 --- a/src/daemon/protocol.h +++ b/src/daemon/protocol.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index 37dffc097..213593aa7 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 015e1e1f9..4ee67f571 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,7 @@ #include "string_tools.h" #include "common/password.h" #include "common/scoped_message_writer.h" +#include "common/pruning.h" #include "daemon/rpc_command_executor.h" #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_core/cryptonote_core.h" @@ -60,13 +61,15 @@ namespace { peer_id_str >> id_str; epee::string_tools::xtype_to_string(peer.port, port_str); std::string addr_str = ip_str + ":" + port_str; - tools::msg_writer() << boost::format("%-10s %-25s %-25s %s") % prefix % id_str % addr_str % elapsed; + std::string rpc_port = peer.rpc_port ? std::to_string(peer.rpc_port) : "-"; + std::string pruning_seed = epee::string_tools::to_string_hex(peer.pruning_seed); + tools::msg_writer() << boost::format("%-10s %-25s %-25s %-5s %-4s %s") % prefix % id_str % addr_str % rpc_port % pruning_seed % elapsed; } void print_block_header(cryptonote::block_header_response const & header) { tools::success_msg_writer() - << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << std::endl + << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" << std::endl << "previous hash: " << header.prev_hash << std::endl << "nonce: " << boost::lexical_cast<std::string>(header.nonce) << std::endl << "is orphan: " << header.orphan_status << std::endl @@ -77,6 +80,7 @@ namespace { << "POW hash: " << header.pow_hash << std::endl << "block size: " << header.block_size << std::endl << "block weight: " << header.block_weight << std::endl + << "long term weight: " << header.long_term_weight << std::endl << "num txes: " << header.num_txes << std::endl << "reward: " << cryptonote::print_money(header.reward); } @@ -154,7 +158,7 @@ t_rpc_command_executor::~t_rpc_command_executor() } } -bool t_rpc_command_executor::print_peer_list() { +bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit) { cryptonote::COMMAND_RPC_GET_PEER_LIST::request req; cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; @@ -175,14 +179,24 @@ bool t_rpc_command_executor::print_peer_list() { } } - for (auto & peer : res.white_list) + if (white) { - print_peer("white", peer); + auto peer = res.white_list.cbegin(); + const auto end = limit ? peer + std::min(limit, res.white_list.size()) : res.white_list.cend(); + for (; peer != end; ++peer) + { + print_peer("white", *peer); + } } - for (auto & peer : res.gray_list) + if (gray) { - print_peer("gray", peer); + auto peer = res.gray_list.cbegin(); + const auto end = limit ? peer + std::min(limit, res.gray_list.size()) : res.gray_list.cend(); + for (; peer != end; ++peer) + { + print_peer("gray", *peer); + } } return true; @@ -449,7 +463,7 @@ bool t_rpc_command_executor::show_status() { % get_sync_percentage(ires) % (ires.testnet ? "testnet" : ires.stagenet ? "stagenet" : "mainnet") % bootstrap_msg - % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed) + std::string(" to ") + mres.address ) : "not mining") + % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed)) : "not mining") % get_mining_speed(ires.difficulty / ires.target) % (unsigned)hfres.version % get_fork_extra_info(hfres.earliest_height, net_height, ires.target) @@ -474,6 +488,81 @@ bool t_rpc_command_executor::show_status() { return true; } +bool t_rpc_command_executor::mining_status() { + cryptonote::COMMAND_RPC_MINING_STATUS::request mreq; + cryptonote::COMMAND_RPC_MINING_STATUS::response mres; + epee::json_rpc::error error_resp; + bool has_mining_info = true; + + std::string fail_message = "Problem fetching info"; + + bool mining_busy = false; + if (m_is_rpc) + { + // mining info is only available non unrestricted RPC mode + has_mining_info = m_rpc_client->rpc_request(mreq, mres, "/mining_status", fail_message.c_str()); + } + else + { + if (!m_rpc_server->on_mining_status(mreq, mres)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + + if (mres.status == CORE_RPC_STATUS_BUSY) + { + mining_busy = true; + } + else if (mres.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, mres.status); + return true; + } + } + + if (!has_mining_info) + { + tools::fail_msg_writer() << "Mining info unavailable"; + return true; + } + + if (mining_busy || !mres.active) + { + tools::msg_writer() << "Not currently mining"; + } + else + { + tools::msg_writer() << "Mining at " << get_mining_speed(mres.speed) << " with " << mres.threads_count << " threads"; + } + + if (mres.active || mres.is_background_mining_enabled) + { + tools::msg_writer() << "PoW algorithm: " << mres.pow_algorithm; + tools::msg_writer() << "Mining address: " << mres.address; + } + + if (mres.is_background_mining_enabled) + { + tools::msg_writer() << "Smart mining enabled:"; + tools::msg_writer() << " Target: " << (unsigned)mres.bg_target << "% CPU"; + tools::msg_writer() << " Idle threshold: " << (unsigned)mres.bg_idle_threshold << "% CPU"; + tools::msg_writer() << " Min idle time: " << (unsigned)mres.bg_min_idle_seconds << " seconds"; + tools::msg_writer() << " Ignore battery: " << (mres.bg_ignore_battery ? "yes" : "no"); + } + + if (!mining_busy && mres.active) + { + uint64_t daily = 86400ull / mres.block_target * mres.block_reward; + uint64_t monthly = 86400ull / mres.block_target * 30.5 * mres.block_reward; + uint64_t yearly = 86400ull / mres.block_target * 356 * mres.block_reward; + tools::msg_writer() << "Expected: " << cryptonote::print_money(daily) << " monero daily, " + << cryptonote::print_money(monthly) << " monero monthly, " << cryptonote::print_money(yearly) << " yearly"; + } + + return true; +} + bool t_rpc_command_executor::print_connections() { cryptonote::COMMAND_RPC_GET_CONNECTIONS::request req; cryptonote::COMMAND_RPC_GET_CONNECTIONS::response res; @@ -498,6 +587,7 @@ bool t_rpc_command_executor::print_connections() { } tools::msg_writer() << std::setw(30) << std::left << "Remote Host" + << std::setw(6) << "SSL" << std::setw(20) << "Peer id" << std::setw(20) << "Support Flags" << std::setw(30) << "Recv/Sent (inactive,sec)" @@ -517,6 +607,7 @@ bool t_rpc_command_executor::print_connections() { tools::msg_writer() //<< std::setw(30) << std::left << in_out << std::setw(30) << std::left << address + << std::setw(6) << (info.ssl ? "yes" : "no") << std::setw(20) << epee::string_tools::pad_string(info.peer_id, 16, '0', true) << std::setw(20) << info.support_flags << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" @@ -569,7 +660,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u if (!first) tools::msg_writer() << "" << std::endl; tools::msg_writer() - << "height: " << header.height << ", timestamp: " << header.timestamp + << "height: " << header.height << ", timestamp: " << header.timestamp << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl @@ -663,7 +754,7 @@ bool t_rpc_command_executor::print_height() { return true; } -bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { +bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -689,13 +780,15 @@ bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; return true; } -bool t_rpc_command_executor::print_block_by_height(uint64_t height) { +bool t_rpc_command_executor::print_block_by_height(uint64_t height, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -721,6 +814,8 @@ bool t_rpc_command_executor::print_block_by_height(uint64_t height) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; @@ -737,6 +832,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash)); req.decode_as_json = false; + req.split = true; req.prune = false; if (m_is_rpc) { @@ -762,13 +858,25 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, if (res.txs.front().in_pool) tools::success_msg_writer() << "Found in pool"; else - tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height; + tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height << (res.txs.front().prunable_as_hex.empty() ? " (pruned)" : ""); } const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front(); + const std::string &pruned_as_hex = (1 == res.txs.size()) ? res.txs.front().pruned_as_hex : ""; + const std::string &prunable_as_hex = (1 == res.txs.size()) ? res.txs.front().prunable_as_hex : ""; // Print raw hex if requested if (include_hex) - tools::success_msg_writer() << as_hex << std::endl; + { + if (!as_hex.empty()) + { + tools::success_msg_writer() << as_hex << std::endl; + } + else + { + std::string output = pruned_as_hex + prunable_as_hex; + tools::success_msg_writer() << output << std::endl; + } + } // Print json if requested if (include_json) @@ -776,17 +884,27 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, crypto::hash tx_hash, tx_prefix_hash; cryptonote::transaction tx; cryptonote::blobdata blob; - if (!string_tools::parse_hexstr_to_binbuff(as_hex, blob)) + std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex; + bool pruned = !pruned_as_hex.empty() && prunable_as_hex.empty(); + if (!string_tools::parse_hexstr_to_binbuff(source, blob)) { tools::fail_msg_writer() << "Failed to parse tx to get json format"; } - else if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash)) - { - tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; - } else { - tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + bool ret; + if (pruned) + ret = cryptonote::parse_and_validate_tx_base_from_blob(blob, tx); + else + ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx); + if (!ret) + { + tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; + } + else + { + tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + } } } } @@ -849,7 +967,7 @@ bool t_rpc_command_executor::print_transaction_pool_long() { } else { - if (!m_rpc_server->on_get_transaction_pool(req, res, false) || res.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_transaction_pool(req, res) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; @@ -935,7 +1053,7 @@ bool t_rpc_command_executor::print_transaction_pool_short() { } else { - if (!m_rpc_server->on_get_transaction_pool(req, res, false) || res.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_transaction_pool(req, res) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; @@ -993,7 +1111,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { else { res.pool_stats = {}; - if (!m_rpc_server->on_get_transaction_pool_stats(req, res, false) || res.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_transaction_pool_stats(req, res) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; @@ -1935,6 +2053,8 @@ bool t_rpc_command_executor::sync_info() for (const auto &p: res.peers) current_download += p.info.current_download; tools::success_msg_writer() << "Downloading at " << current_download << " kB/s"; + if (res.next_needed_pruning_seed) + tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed; tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; for (const auto &p: res.peers) @@ -1942,25 +2062,30 @@ bool t_rpc_command_executor::sync_info() std::string address = epee::string_tools::pad_string(p.info.address, 24); uint64_t nblocks = 0, size = 0; for (const auto &s: res.spans) - if (s.rate > 0.0f && s.connection_id == p.info.connection_id) + if (s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << epee::string_tools::pad_string(p.info.state, 16) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << + epee::string_tools::pad_string(p.info.state, 16) << " " << + epee::string_tools::pad_string(epee::string_tools::to_string_hex(p.info.pruning_seed), 8) << " " << p.info.height << " " << + p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; for (const auto &s: res.spans) total_size += s.size; tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB"; + tools::success_msg_writer() << res.overview; for (const auto &s: res.spans) { std::string address = epee::string_tools::pad_string(s.remote_address, 24); + std::string pruning_seed = epee::string_tools::to_string_hex(tools::get_pruning_seed(s.start_block_height, std::numeric_limits<uint64_t>::max(), CRYPTONOTE_PRUNING_LOG_STRIPES)); if (s.size == 0) { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; } else { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; } } @@ -1994,4 +2119,69 @@ bool t_rpc_command_executor::pop_blocks(uint64_t num_blocks) return true; } +bool t_rpc_command_executor::prune_blockchain() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = false; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + tools::success_msg_writer() << "Blockchain pruned: seed " << epee::string_tools::to_string_hex(res.pruning_seed); + return true; +} + +bool t_rpc_command_executor::check_blockchain_pruning() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = true; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + if (res.pruning_seed) + { + tools::success_msg_writer() << "Blockchain pruning checked"; + } + else + { + tools::success_msg_writer() << "Blockchain is not pruned"; + } + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 592584a5f..423132b79 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -6,7 +6,7 @@ */ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -67,7 +67,7 @@ public: ~t_rpc_command_executor(); - bool print_peer_list(); + bool print_peer_list(bool white = true, bool gray = true, size_t limit = 0); bool print_peer_list_stats(); @@ -91,9 +91,9 @@ public: bool print_height(); - bool print_block_by_hash(crypto::hash block_hash); + bool print_block_by_hash(crypto::hash block_hash, bool include_hex); - bool print_block_by_height(uint64_t height); + bool print_block_by_height(uint64_t height, bool include_hex); bool print_transaction(crypto::hash transaction_hash, bool include_hex, bool include_json); @@ -109,6 +109,8 @@ public: bool stop_mining(); + bool mining_status(); + bool stop_daemon(); bool print_status(); @@ -154,6 +156,10 @@ public: bool sync_info(); bool pop_blocks(uint64_t num_blocks); + + bool prune_blockchain(); + + bool check_blockchain_pruning(); }; } // namespace daemonize diff --git a/src/daemonizer/CMakeLists.txt b/src/daemonizer/CMakeLists.txt index 2753d0003..d540adc10 100644 --- a/src/daemonizer/CMakeLists.txt +++ b/src/daemonizer/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/daemonizer/daemonizer.h b/src/daemonizer/daemonizer.h index c5852b59c..a45a2b44f 100644 --- a/src/daemonizer/daemonizer.h +++ b/src/daemonizer/daemonizer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/posix_daemonizer.inl b/src/daemonizer/posix_daemonizer.inl index b3f3f262f..9ec09c678 100644 --- a/src/daemonizer/posix_daemonizer.inl +++ b/src/daemonizer/posix_daemonizer.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/posix_fork.h b/src/daemonizer/posix_fork.h index 9294b00e2..27b4ac18d 100644 --- a/src/daemonizer/posix_fork.h +++ b/src/daemonizer/posix_fork.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_daemonizer.inl b/src/daemonizer/windows_daemonizer.inl index 7e61e3603..701c098f6 100644 --- a/src/daemonizer/windows_daemonizer.inl +++ b/src/daemonizer/windows_daemonizer.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp index 1302fa578..7d53b07c7 100644 --- a/src/daemonizer/windows_service.cpp +++ b/src/daemonizer/windows_service.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_service.h b/src/daemonizer/windows_service.h index aacf3d039..12429c78a 100644 --- a/src/daemonizer/windows_service.h +++ b/src/daemonizer/windows_service.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/daemonizer/windows_service_runner.h b/src/daemonizer/windows_service_runner.h index 06e180823..703795ce9 100644 --- a/src/daemonizer/windows_service_runner.h +++ b/src/daemonizer/windows_service_runner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/debug_utilities/CMakeLists.txt b/src/debug_utilities/CMakeLists.txt index 1bcbfd0cf..7bc2c324f 100644 --- a/src/debug_utilities/CMakeLists.txt +++ b/src/debug_utilities/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 83422083b..89c9db02e 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -80,11 +80,9 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true}; const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; const command_line::arg_descriptor<std::string> arg_input = {"input", "Specify input has a hexadecimal string", ""}; - command_line::add_arg(desc_cmd_sett, arg_output_file); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_input); @@ -120,52 +118,10 @@ int main(int argc, char* argv[]) mlog_configure("", true); - std::string m_config_folder; - - std::ostream *output; - std::ofstream *raw_data_file = NULL; - if (command_line::has_arg(vm, arg_output_file)) - { - output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); - - const boost::filesystem::path dir_path = output_file_path.parent_path(); - if (!dir_path.empty()) - { - if (boost::filesystem::exists(dir_path)) - { - if (!boost::filesystem::is_directory(dir_path)) - { - std::cerr << "output directory path is a file: " << dir_path << std::endl; - return 1; - } - } - else - { - if (!boost::filesystem::create_directory(dir_path)) - { - std::cerr << "Failed to create directory " << dir_path << std::endl; - return 1; - } - } - } - - raw_data_file = new std::ofstream(); - raw_data_file->open(output_file_path.string(), std::ios_base::out | std::ios::trunc); - if (raw_data_file->fail()) - return 1; - output = raw_data_file; - } - else - { - output_file_path = ""; - output = &std::cout; - } - cryptonote::blobdata blob; if (!epee::string_tools::parse_hexstr_to_binbuff(input, blob)) { std::cerr << "Invalid hex input" << std::endl; - std::cerr << "Invalid hex input: " << input << std::endl; return 1; } @@ -180,11 +136,9 @@ int main(int argc, char* argv[]) } else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { -/* if (tx.pruned) std::cout << "Parsed pruned transaction:" << std::endl; else -*/ std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; @@ -214,11 +168,5 @@ int main(int argc, char* argv[]) - if (output->fail()) - return 1; - output->flush(); - if (raw_data_file) - delete raw_data_file; - return 0; } diff --git a/src/debug_utilities/object_sizes.cpp b/src/debug_utilities/object_sizes.cpp index 2281d0734..302f0a206 100644 --- a/src/debug_utilities/object_sizes.cpp +++ b/src/debug_utilities/object_sizes.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index 91d670b73..ffa1458b0 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2017, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/device/device.cpp b/src/device/device.cpp index d5e3031ff..fbd77dab9 100644 --- a/src/device/device.cpp +++ b/src/device/device.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/device/device.hpp b/src/device/device.hpp index 399648f01..7f1a60c9f 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -69,6 +69,7 @@ namespace cryptonote struct account_public_address; struct account_keys; struct subaddress_index; + struct tx_destination_entry; } namespace hw { @@ -208,12 +209,15 @@ namespace hw { return encrypt_payment_id(payment_id, public_key, secret_key); } - virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) = 0; - virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) = 0; - - virtual bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) = 0; + virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) = 0; + virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) = 0; + virtual bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys, + std::vector<crypto::public_key> &additional_tx_public_keys, + std::vector<rct::key> &amount_keys, + crypto::public_key &out_eph_public_key) = 0; virtual bool mlsag_prehash(const std::string &blob, size_t inputs_size, size_t outputs_size, const rct::keyV &hashes, const rct::ctkeyV &outPk, rct::key &prehash) = 0; virtual bool mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &rvII) = 0; diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index 22128cec1..8b8cdf6d2 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 2286998a4..999fbc22f 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -34,8 +34,10 @@ #include "int-util.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/subaddress_index.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #include "ringct/rctOps.h" +#include "log.hpp" #define ENCRYPTED_PAYMENT_ID_TAIL 0x8d #define CHACHA8_KEY_TAIL 0x8c @@ -278,10 +280,55 @@ namespace hw { return true; } + bool device_default::generate_output_ephemeral_keys(const size_t tx_version, + const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys, + std::vector<crypto::public_key> &additional_tx_public_keys, + std::vector<rct::key> &amount_keys, crypto::public_key &out_eph_public_key) { - bool device_default::add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) { - return true; + crypto::key_derivation derivation; + + // make additional tx pubkey if necessary + cryptonote::keypair additional_txkey; + if (need_additional_txkeys) + { + additional_txkey.sec = additional_tx_keys[output_index]; + if (dst_entr.is_subaddress) + additional_txkey.pub = rct::rct2pk(rct::scalarmultKey(rct::pk2rct(dst_entr.addr.m_spend_public_key), rct::sk2rct(additional_txkey.sec))); + else + additional_txkey.pub = rct::rct2pk(rct::scalarmultBase(rct::sk2rct(additional_txkey.sec))); + } + + bool r; + if (change_addr && dst_entr.addr == *change_addr) + { + // sending change to yourself; derivation = a*R + r = generate_key_derivation(txkey_pub, sender_account_keys.m_view_secret_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << txkey_pub << ", " << sender_account_keys.m_view_secret_key << ")"); + } + else + { + // sending to the recipient; derivation = r*A (or s*C in the subaddress scheme) + r = generate_key_derivation(dst_entr.addr.m_view_public_key, dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << dst_entr.addr.m_view_public_key << ", " << (dst_entr.is_subaddress && need_additional_txkeys ? additional_txkey.sec : tx_key) << ")"); + } + + if (need_additional_txkeys) + { + additional_tx_public_keys.push_back(additional_txkey.pub); + } + + if (tx_version > 1) + { + crypto::secret_key scalar1; + derivation_to_scalar(derivation, output_index, scalar1); + amount_keys.push_back(rct::sk2rct(scalar1)); + } + r = derive_public_key(derivation, output_index, dst_entr.addr.m_spend_public_key, out_eph_public_key); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << output_index << ", "<< dst_entr.addr.m_spend_public_key << ")"); + + return r; } bool device_default::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) { @@ -302,13 +349,13 @@ namespace hw { return true; } - bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) { - rct::ecdhEncode(unmasked, sharedSec); + bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhEncode(unmasked, sharedSec, short_amount); return true; } - bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) { - rct::ecdhDecode(masked, sharedSec); + bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhDecode(masked, sharedSec, short_amount); return true; } diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 5c59a9066..90d39495b 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -111,12 +111,15 @@ namespace hw { bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; - - bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) override; + bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys, + std::vector<crypto::public_key> &additional_tx_public_keys, + std::vector<rct::key> &amount_keys, + crypto::public_key &out_eph_public_key) override; bool mlsag_prehash(const std::string &blob, size_t inputs_size, size_t outputs_size, const rct::keyV &hashes, const rct::ctkeyV &outPk, rct::key &prehash) override; bool mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &rvII) override; diff --git a/src/device/device_io.hpp b/src/device/device_io.hpp index 96163a211..fe66736f7 100644 --- a/src/device/device_io.hpp +++ b/src/device/device_io.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -50,7 +50,7 @@ namespace hw { virtual void disconnect() = 0; virtual bool connected() const = 0; - virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) = 0; + virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) = 0; }; }; }; diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp index 1aadfb9ea..f07e0eaae 100644 --- a/src/device/device_io_hid.cpp +++ b/src/device/device_io_hid.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -148,7 +148,7 @@ namespace hw { return this->usb_device != NULL; } - int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) { + int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) { unsigned char buffer[400]; unsigned char padding_buffer[MAX_BLOCK+1]; unsigned int result; @@ -177,7 +177,11 @@ namespace hw { //get first response memset(buffer, 0, sizeof(buffer)); - hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + if (!user_input) { + hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + } else { + hid_ret = hid_read(this->usb_device, buffer, MAX_BLOCK); + } ASSERT_X(hid_ret>=0, "Unable to read hidapi response. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); result = (unsigned int)hid_ret; io_hid_log(1, buffer, result); diff --git a/src/device/device_io_hid.hpp b/src/device/device_io_hid.hpp index bb0f0a814..ed22058d6 100644 --- a/src/device/device_io_hid.hpp +++ b/src/device/device_io_hid.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -100,7 +100,7 @@ namespace hw { void connect(void *params); void connect(unsigned int vid, unsigned int pid, boost::optional<int> interface_number, boost::optional<unsigned short> usage_page); bool connected() const; - int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len); + int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input); void disconnect(); void release(); }; diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index bfb41bbe4..0f197272c 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -32,6 +32,7 @@ #include "ringct/rctOps.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/subaddress_index.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #include <boost/thread/locks.hpp> #include <boost/thread/lock_guard.hpp> @@ -67,10 +68,12 @@ namespace hw { /* === Keymap ==== */ /* ===================================================================== */ - ABPkeys::ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, const size_t real_output_index, const rct::key& P, const rct::key& AK) { + ABPkeys::ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, const bool is_change, const bool need_additional_txkeys, const size_t real_output_index, const rct::key& P, const rct::key& AK) { Aout = A; Bout = B; is_subaddress = is_subaddr; + is_change_address = is_change; + additional_key = need_additional_txkeys; index = real_output_index; Pout = P; AKout = AK; @@ -80,6 +83,8 @@ namespace hw { Aout = keys.Aout; Bout = keys.Bout; is_subaddress = keys.is_subaddress; + is_change_address = keys.is_change_address; + additional_key = keys.additional_key; index = keys.index; Pout = keys.Pout; AKout = keys.AKout; @@ -137,6 +142,8 @@ namespace hw { static int device_id = 0; + #define PROTOCOL_VERSION 2 + #define INS_NONE 0x00 #define INS_RESET 0x02 @@ -168,6 +175,7 @@ namespace hw { #define INS_STEALTH 0x76 #define INS_BLIND 0x78 #define INS_UNBLIND 0x7A + #define INS_GEN_TXOUT_KEYS 0x7B #define INS_VALIDATE 0x7C #define INS_MLSAG 0x7E #define INS_CLOSE_TX 0x80 @@ -176,7 +184,7 @@ namespace hw { #define INS_GET_RESPONSE 0xc0 - device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 120000) { + device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 2000) { this->id = device_id++; this->reset_buffer(); this->mode = NONE; @@ -235,6 +243,9 @@ namespace hw { /* IO */ /* ======================================================================= */ + #define IO_SW_DENY 0x6982 + #define IO_SECRET_KEY 0x02 + void device_ledger::logCMD() { if (apdu_verbose) { char strbuffer[1024]; @@ -264,8 +275,7 @@ namespace hw { int device_ledger::set_command_header(unsigned char ins, unsigned char p1, unsigned char p2) { reset_buffer(); - int offset = 0; - this->buffer_send[0] = 0x00; + this->buffer_send[0] = PROTOCOL_VERSION; this->buffer_send[1] = ins; this->buffer_send[2] = p1; this->buffer_send[3] = p2; @@ -283,7 +293,12 @@ namespace hw { void device_ledger::send_simple(unsigned char ins, unsigned char p1) { this->length_send = set_command_header_noopt(ins, p1); - this->exchange(); + if (ins == INS_GET_KEY && p1 == IO_SECRET_KEY) { + // export view key user input + this->exchange_wait_on_input(); + } else { + this->exchange(); + } } bool device_ledger::reset() { @@ -294,7 +309,7 @@ namespace hw { unsigned int device_ledger::exchange(unsigned int ok, unsigned int mask) { logCMD(); - this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE); + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, false); ASSERT_X(this->length_recv>=2, "Communication error, less than tow bytes received"); this->length_recv -= 2; @@ -305,6 +320,25 @@ namespace hw { return this->sw; } + unsigned int device_ledger::exchange_wait_on_input(unsigned int ok, unsigned int mask) { + logCMD(); + unsigned int deny = 0; + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, true); + ASSERT_X(this->length_recv>=2, "Communication error, less than two bytes received"); + + this->length_recv -= 2; + this->sw = (this->buffer_recv[length_recv]<<8) | this->buffer_recv[length_recv+1]; + if (this->sw == IO_SW_DENY) { + // cancel on device + deny = 1; + } else { + ASSERT_SW(this->sw,ok,msk); + } + + logRESP(); + return deny; + } + void device_ledger::reset_buffer() { this->length_send = 0; memset(this->buffer_send, 0, BUFFER_SEND_SIZE); @@ -322,7 +356,7 @@ namespace hw { } const std::string device_ledger::get_name() const { - if (this->full_name.empty() || !this->connected()) { + if (!this->connected()) { return std::string("<disconnected:").append(this->name).append(">"); } return this->name; @@ -481,11 +515,11 @@ namespace hw { } const std::size_t output_index_x = output_index; crypto::public_key derived_pub_x; - hw::ledger::log_hexbuffer("derive_subaddress_public_key: [[IN]] pub ", pub_x.data, 32); - hw::ledger::log_hexbuffer("derive_subaddress_public_key: [[IN]] derivation", derivation_x.data, 32); - hw::ledger::log_message ("derive_subaddress_public_key: [[IN]] index ", std::to_string((int)output_index_x)); + log_hexbuffer("derive_subaddress_public_key: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("derive_subaddress_public_key: [[IN]] derivation", derivation_x.data, 32); + log_message ("derive_subaddress_public_key: [[IN]] index ", std::to_string((int)output_index_x)); this->controle_device->derive_subaddress_public_key(pub_x, derivation_x,output_index_x,derived_pub_x); - hw::ledger::log_hexbuffer("derive_subaddress_public_key: [[OUT]] derived_pub", derived_pub_x.data, 32); + log_hexbuffer("derive_subaddress_public_key: [[OUT]] derived_pub", derived_pub_x.data, 32); #endif if ((this->mode == TRANSACTION_PARSE) && has_view_key) { @@ -531,11 +565,11 @@ namespace hw { const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); const cryptonote::subaddress_index index_x = index; crypto::public_key D_x; - hw::ledger::log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data,32); - hw::ledger::log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_spend_secret_key", keys_x.m_spend_secret_key.data,32); - hw::ledger::log_message ("get_subaddress_spend_public_key: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); + log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data,32); + log_hexbuffer("get_subaddress_spend_public_key: [[IN]] keys.m_spend_secret_key", keys_x.m_spend_secret_key.data,32); + log_message ("get_subaddress_spend_public_key: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); D_x = this->controle_device->get_subaddress_spend_public_key(keys_x, index_x); - hw::ledger::log_hexbuffer("get_subaddress_spend_public_key: [[OUT]] derivation ", D_x.data, 32); + log_hexbuffer("get_subaddress_spend_public_key: [[OUT]] derivation ", D_x.data, 32); #endif if (index.is_zero()) { @@ -582,14 +616,14 @@ namespace hw { const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); const cryptonote::subaddress_index index_x = index; cryptonote::account_public_address address_x; - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_view_public_key", keys_x.m_account_address.m_view_public_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[IN]] keys.m_spend_public_key", keys_x.m_account_address.m_spend_public_key.data, 32); - hw::ledger::log_message ("get_subaddress: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); + log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); + log_hexbuffer("get_subaddress: [[IN]] keys.m_view_public_key", keys_x.m_account_address.m_view_public_key.data, 32); + log_hexbuffer("get_subaddress: [[IN]] keys.m_view_secret_key ", keys_x.m_view_secret_key.data, 32); + log_hexbuffer("get_subaddress: [[IN]] keys.m_spend_public_key", keys_x.m_account_address.m_spend_public_key.data, 32); + log_message ("get_subaddress: [[IN]] index ", std::to_string(index_x.major)+"."+std::to_string(index_x.minor)); address_x = this->controle_device->get_subaddress(keys_x, index_x); - hw::ledger::log_hexbuffer("get_subaddress: [[OUT]] keys.m_view_public_key ", address_x.m_view_public_key.data, 32); - hw::ledger::log_hexbuffer("get_subaddress: [[OUT]] keys.m_spend_public_key", address_x.m_spend_public_key.data, 32); + log_hexbuffer("get_subaddress: [[OUT]] keys.m_view_public_key ", address_x.m_view_public_key.data, 32); + log_hexbuffer("get_subaddress: [[OUT]] keys.m_spend_public_key", address_x.m_spend_public_key.data, 32); #endif if (index.is_zero()) { @@ -625,10 +659,10 @@ namespace hw { const crypto::secret_key sec_x = hw::ledger::decrypt(sec); const cryptonote::subaddress_index index_x = index; crypto::secret_key sub_sec_x; - hw::ledger::log_message ("get_subaddress_secret_key: [[IN]] index ", std::to_string(index.major)+"."+std::to_string(index.minor)); - hw::ledger::log_hexbuffer("get_subaddress_secret_key: [[IN]] sec ", sec_x.data, 32); + log_message ("get_subaddress_secret_key: [[IN]] index ", std::to_string(index.major)+"."+std::to_string(index.minor)); + log_hexbuffer("get_subaddress_secret_key: [[IN]] sec ", sec_x.data, 32); sub_sec_x = this->controle_device->get_subaddress_secret_key(sec_x, index_x); - hw::ledger::log_hexbuffer("get_subaddress_secret_key: [[OUT]] sub_sec", sub_sec_x.data, 32); + log_hexbuffer("get_subaddress_secret_key: [[OUT]] sub_sec", sub_sec_x.data, 32); #endif int offset = set_command_header_noopt(INS_GET_SUBADDRESS_SECRET_KEY); @@ -660,7 +694,7 @@ namespace hw { bool device_ledger::verify_keys(const crypto::secret_key &secret_key, const crypto::public_key &public_key) { AUTO_LOCK_CMD(); - int offset, sw; + int offset; offset = set_command_header_noopt(INS_VERIFY_KEY); //sec @@ -690,10 +724,10 @@ namespace hw { const rct::key P_x = P; const rct::key a_x = hw::ledger::decrypt(a); rct::key aP_x; - hw::ledger::log_hexbuffer("scalarmultKey: [[IN]] P ", (char*)P_x.bytes, 32); - hw::ledger::log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[IN]] P ", (char*)P_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); this->controle_device->scalarmultKey(aP_x, P_x, a_x); - hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aP", (char*)aP_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[OUT]] aP", (char*)aP_x.bytes, 32); #endif int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_KEY); @@ -725,9 +759,9 @@ namespace hw { #ifdef DEBUG_HWDEVICE const rct::key a_x = hw::ledger::decrypt(a); rct::key aG_x; - hw::ledger::log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[IN]] a ", (char*)a_x.bytes, 32); this->controle_device->scalarmultBase(aG_x, a_x); - hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aG", (char*)aG_x.bytes, 32); + log_hexbuffer("scalarmultKey: [[OUT]] aG", (char*)aG_x.bytes, 32); #endif int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_BASE); @@ -818,10 +852,10 @@ namespace hw { const crypto::public_key pub_x = pub; const crypto::secret_key sec_x = hw::ledger::decrypt(sec); crypto::key_derivation derivation_x; - hw::ledger::log_hexbuffer("generate_key_derivation: [[IN]] pub ", pub_x.data, 32); - hw::ledger::log_hexbuffer("generate_key_derivation: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("generate_key_derivation: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("generate_key_derivation: [[IN]] sec ", sec_x.data, 32); this->controle_device->generate_key_derivation(pub_x, sec_x, derivation_x); - hw::ledger::log_hexbuffer("generate_key_derivation: [[OUT]] derivation", derivation_x.data, 32); + log_hexbuffer("generate_key_derivation: [[OUT]] derivation", derivation_x.data, 32); #endif if ((this->mode == TRANSACTION_PARSE) && has_view_key) { @@ -887,10 +921,10 @@ namespace hw { const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); const size_t output_index_x = output_index; crypto::ec_scalar res_x; - hw::ledger::log_hexbuffer("derivation_to_scalar: [[IN]] derivation ", derivation_x.data, 32); - hw::ledger::log_message ("derivation_to_scalar: [[IN]] output_index ", std::to_string(output_index_x)); + log_hexbuffer("derivation_to_scalar: [[IN]] derivation ", derivation_x.data, 32); + log_message ("derivation_to_scalar: [[IN]] output_index ", std::to_string(output_index_x)); this->controle_device->derivation_to_scalar(derivation_x, output_index_x, res_x); - hw::ledger::log_hexbuffer("derivation_to_scalar: [[OUT]] res ", res_x.data, 32); + log_hexbuffer("derivation_to_scalar: [[OUT]] res ", res_x.data, 32); #endif int offset = set_command_header_noopt(INS_DERIVATION_TO_SCALAR); @@ -927,11 +961,11 @@ namespace hw { const std::size_t output_index_x = output_index; const crypto::secret_key sec_x = hw::ledger::decrypt(sec); crypto::secret_key derived_sec_x; - hw::ledger::log_hexbuffer("derive_secret_key: [[IN]] derivation ", derivation_x.data, 32); - hw::ledger::log_message ("derive_secret_key: [[IN]] index ", std::to_string(output_index_x)); - hw::ledger::log_hexbuffer("derive_secret_key: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("derive_secret_key: [[IN]] derivation ", derivation_x.data, 32); + log_message ("derive_secret_key: [[IN]] index ", std::to_string(output_index_x)); + log_hexbuffer("derive_secret_key: [[IN]] sec ", sec_x.data, 32); this->controle_device->derive_secret_key(derivation_x, output_index_x, sec_x, derived_sec_x); - hw::ledger::log_hexbuffer("derive_secret_key: [[OUT]] derived_sec", derived_sec_x.data, 32); + log_hexbuffer("derive_secret_key: [[OUT]] derived_sec", derived_sec_x.data, 32); #endif int offset = set_command_header_noopt(INS_DERIVE_SECRET_KEY); @@ -971,11 +1005,11 @@ namespace hw { const std::size_t output_index_x = output_index; const crypto::public_key pub_x = pub; crypto::public_key derived_pub_x; - hw::ledger::log_hexbuffer("derive_public_key: [[IN]] derivation ", derivation_x.data, 32); - hw::ledger::log_message ("derive_public_key: [[IN]] output_index", std::to_string(output_index_x)); - hw::ledger::log_hexbuffer("derive_public_key: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("derive_public_key: [[IN]] derivation ", derivation_x.data, 32); + log_message ("derive_public_key: [[IN]] output_index", std::to_string(output_index_x)); + log_hexbuffer("derive_public_key: [[IN]] pub ", pub_x.data, 32); this->controle_device->derive_public_key(derivation_x, output_index_x, pub_x, derived_pub_x); - hw::ledger::log_hexbuffer("derive_public_key: [[OUT]] derived_pub ", derived_pub_x.data, 32); + log_hexbuffer("derive_public_key: [[OUT]] derived_pub ", derived_pub_x.data, 32); #endif int offset = set_command_header_noopt(INS_DERIVE_PUBLIC_KEY); @@ -1012,11 +1046,11 @@ namespace hw { #ifdef DEBUG_HWDEVICE const crypto::secret_key sec_x = hw::ledger::decrypt(sec); crypto::public_key pub_x; - hw::ledger::log_hexbuffer("secret_key_to_public_key: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("secret_key_to_public_key: [[IN]] sec ", sec_x.data, 32); bool rc = this->controle_device->secret_key_to_public_key(sec_x, pub_x); - hw::ledger::log_hexbuffer("secret_key_to_public_key: [[OUT]] pub", pub_x.data, 32); + log_hexbuffer("secret_key_to_public_key: [[OUT]] pub", pub_x.data, 32); if (!rc){ - hw::ledger::log_message("secret_key_to_public_key", "secret_key rejected"); + log_message("secret_key_to_public_key", "secret_key rejected"); } #endif @@ -1046,10 +1080,10 @@ namespace hw { const crypto::public_key pub_x = pub; const crypto::secret_key sec_x = hw::ledger::decrypt(sec); crypto::key_image image_x; - hw::ledger::log_hexbuffer("generate_key_image: [[IN]] pub ", pub_x.data, 32); - hw::ledger::log_hexbuffer("generate_key_image: [[IN]] sec ", sec_x.data, 32); + log_hexbuffer("generate_key_image: [[IN]] pub ", pub_x.data, 32); + log_hexbuffer("generate_key_image: [[IN]] sec ", sec_x.data, 32); this->controle_device->generate_key_image(pub_x, sec_x, image_x); - hw::ledger::log_hexbuffer("generate_key_image: [[OUT]] image ", image_x.data, 32); + log_hexbuffer("generate_key_image: [[OUT]] image ", image_x.data, 32); #endif int offset = set_command_header_noopt(INS_GEN_KEY_IMAGE); @@ -1133,23 +1167,155 @@ namespace hw { return true; } - bool device_ledger::add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) { - AUTO_LOCK_CMD(); - key_map.add(ABPkeys(rct::pk2rct(Aout),rct::pk2rct(Bout), is_subaddress, real_output_index, rct::pk2rct(out_eph_public_key), amount_key)); + + bool device_ledger::generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys, + std::vector<crypto::public_key> &additional_tx_public_keys, + std::vector<rct::key> &amount_keys, + crypto::public_key &out_eph_public_key) { + AUTO_LOCK_CMD(); + + #ifdef DEBUG_HWDEVICE + const size_t &tx_version_x = tx_version; + const cryptonote::account_keys sender_account_keys_x = sender_account_keys; + memmove((void*)sender_account_keys_x.m_view_secret_key.data, dbg_viewkey.data, 32); + + const crypto::public_key &txkey_pub_x = txkey_pub; + const crypto::secret_key &tx_key_x = tx_key; + const cryptonote::tx_destination_entry &dst_entr_x = dst_entr; + const boost::optional<cryptonote::account_public_address> &change_addr_x = change_addr; + const size_t &output_index_x = output_index; + const bool &need_additional_txkeys_x = need_additional_txkeys; + const std::vector<crypto::secret_key> &additional_tx_keys_x = additional_tx_keys; + std::vector<crypto::public_key> additional_tx_public_keys_x; + std::vector<rct::key> amount_keys_x; + crypto::public_key out_eph_public_key_x; + this->controle_device->generate_output_ephemeral_keys(tx_version_x, sender_account_keys_x, txkey_pub_x, tx_key_x, dst_entr_x, change_addr_x, output_index_x, need_additional_txkeys_x, additional_tx_keys_x, + additional_tx_public_keys_x, amount_keys_x, out_eph_public_key_x); + #endif + + // make additional tx pubkey if necessary + cryptonote::keypair additional_txkey; + if (need_additional_txkeys) { + additional_txkey.sec = additional_tx_keys[output_index]; + } + + //compute derivation, out_eph_public_key, and amount key in one shot on device, to ensure checkable link + const crypto::secret_key *sec; + bool is_change; + + if (change_addr && dst_entr.addr == *change_addr) + { + // sending change to yourself; derivation = a*R + is_change = true; + sec = &sender_account_keys.m_view_secret_key; + } + else + { + is_change = false; + if (dst_entr.is_subaddress && need_additional_txkeys) { + sec = &additional_txkey.sec; + } else { + sec = &tx_key; + } + } + + int offset = set_command_header_noopt(INS_GEN_TXOUT_KEYS); + //tx_version + this->buffer_send[offset+0] = tx_version>>24; + this->buffer_send[offset+1] = tx_version>>16; + this->buffer_send[offset+2] = tx_version>>8; + this->buffer_send[offset+3] = tx_version>>0; + offset += 4; + //tx_sec + memmove(&this->buffer_send[offset], sec->data, 32); + offset += 32; + //Aout + memmove(&this->buffer_send[offset], dst_entr.addr.m_view_public_key.data, 32); + offset += 32; + //Bout + memmove(&this->buffer_send[offset], dst_entr.addr.m_spend_public_key.data, 32); + offset += 32; + //output index + this->buffer_send[offset+0] = output_index>>24; + this->buffer_send[offset+1] = output_index>>16; + this->buffer_send[offset+2] = output_index>>8; + this->buffer_send[offset+3] = output_index>>0; + offset += 4; + //is_change, + this->buffer_send[offset] = is_change; + offset++; + //is_subaddress + this->buffer_send[offset] = dst_entr.is_subaddress; + offset++; + //need_additional_key + this->buffer_send[offset] = need_additional_txkeys; + offset++; + + this->buffer_send[4] = offset-5; + this->length_send = offset; + this->exchange(); + + offset = 0; + unsigned int recv_len = this->length_recv; + if (need_additional_txkeys) + { + ASSERT_X(recv_len>=32, "Not enought data from device"); + memmove(additional_txkey.pub.data, &this->buffer_recv[offset], 32); + additional_tx_public_keys.push_back(additional_txkey.pub); + offset += 32; + recv_len -= 32; + } + if (tx_version > 1) + { + ASSERT_X(recv_len>=32, "Not enought data from device"); + crypto::secret_key scalar1; + memmove(scalar1.data, &this->buffer_recv[offset],32); + amount_keys.push_back(rct::sk2rct(scalar1)); + offset += 32; + recv_len -= 32; + } + ASSERT_X(recv_len>=32, "Not enought data from device"); + memmove(out_eph_public_key.data, &this->buffer_recv[offset], 32); + recv_len -= 32; + + // add ABPkeys + this->add_output_key_mapping(dst_entr.addr.m_view_public_key, dst_entr.addr.m_spend_public_key, dst_entr.is_subaddress, is_change, + need_additional_txkeys, output_index, + amount_keys.back(), out_eph_public_key); + + #ifdef DEBUG_HWDEVICE + hw::ledger::check32("generate_output_ephemeral_keys", "amount_key", (const char*)amount_keys_x.back().bytes, (const char*)hw::ledger::decrypt(amount_keys.back()).bytes); + if (need_additional_txkeys) { + hw::ledger::check32("generate_output_ephemeral_keys", "additional_tx_key", additional_tx_keys_x.back().data, additional_tx_keys.back().data); + } + hw::ledger::check32("generate_output_ephemeral_keys", "out_eph_public_key", out_eph_public_key_x.data, out_eph_public_key.data); + #endif + + return true; + } + + bool device_ledger::add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const bool is_change, + const bool need_additional, const size_t real_output_index, + const rct::key &amount_key, const crypto::public_key &out_eph_public_key) { + key_map.add(ABPkeys(rct::pk2rct(Aout),rct::pk2rct(Bout), is_subaddress, is_change, need_additional, real_output_index, rct::pk2rct(out_eph_public_key), amount_key)); return true; } - bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout) { + bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple unmasked_x = unmasked; - this->controle_device->ecdhEncode(unmasked_x, AKout_x); + this->controle_device->ecdhEncode(unmasked_x, AKout_x, short_amount); #endif - int offset = set_command_header_noopt(INS_BLIND); + int offset = set_command_header(INS_BLIND); + //options + this->buffer_send[offset] = short_amount?0x02:0x00; + offset += 1; // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1171,23 +1337,25 @@ namespace hw { hw::ledger::check32("ecdhEncode", "amount", (char*)unmasked_x.amount.bytes, (char*)unmasked.amount.bytes); hw::ledger::check32("ecdhEncode", "mask", (char*)unmasked_x.mask.bytes, (char*)unmasked.mask.bytes); - hw::ledger::log_hexbuffer("Blind AKV input", (char*)&this->buffer_recv[64], 3*32); + log_hexbuffer("Blind AKV input", (char*)&this->buffer_recv[64], 3*32); #endif return true; } - bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout) { + bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple masked_x = masked; - this->controle_device->ecdhDecode(masked_x, AKout_x); + this->controle_device->ecdhDecode(masked_x, AKout_x, short_amount); #endif - int offset = set_command_header_noopt(INS_UNBLIND); - + int offset = set_command_header(INS_UNBLIND); + //options + this->buffer_send[offset] = short_amount?0x02:0x00; + offset += 1; // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1260,7 +1428,8 @@ namespace hw { this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check fee user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Fee denied on device."); //pseudoOuts if (type == rct::RCTTypeSimple) { @@ -1282,7 +1451,11 @@ namespace hw { // ====== Aout, Bout, AKout, C, v, k ====== kv_offset = data_offset; - C_offset = kv_offset+ (32*2)*outputs_size; + if (type==rct::RCTTypeBulletproof2) { + C_offset = kv_offset+ (8)*outputs_size; + } else { + C_offset = kv_offset+ (32+32)*outputs_size; + } for ( i = 0; i < outputs_size; i++) { ABPkeys outKeys; bool found; @@ -1295,11 +1468,15 @@ namespace hw { offset = set_command_header(INS_VALIDATE, 0x02, i+1); //options this->buffer_send[offset] = (i==outputs_size-1)? 0x00:0x80 ; + this->buffer_send[offset] |= (type==rct::RCTTypeBulletproof2)?0x02:0x00; offset += 1; if (found) { //is_subaddress this->buffer_send[offset] = outKeys.is_subaddress; offset++; + //is_change_address + this->buffer_send[offset] = outKeys.is_change_address; + offset++; //Aout memmove(this->buffer_send+offset, outKeys.Aout.bytes, 32); offset+=32; @@ -1311,26 +1488,38 @@ namespace hw { offset+=32; } else { // dummy: is_subaddress Aout Bout AKout - offset += 1+32*3; + offset += 2+32*3; } //C memmove(this->buffer_send+offset, data+C_offset,32); offset += 32; C_offset += 32; - //k - memmove(this->buffer_send+offset, data+kv_offset,32); - offset += 32; - //v - kv_offset += 32; - memmove(this->buffer_send+offset, data+kv_offset,32); - offset += 32; - kv_offset += 32; + if (type==rct::RCTTypeBulletproof2) { + //k + memset(this->buffer_send+offset, 0, 32); + offset += 32; + //v + memset(this->buffer_send+offset, 0, 32); + memmove(this->buffer_send+offset, data+kv_offset,8); + offset += 32; + kv_offset += 8; + } else { + //k + memmove(this->buffer_send+offset, data+kv_offset,32); + offset += 32; + kv_offset += 32; + //v + memmove(this->buffer_send+offset, data+kv_offset,32); + offset += 32; + kv_offset += 32; + } this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check transaction user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Transaction denied on device."); #ifdef DEBUG_HWDEVICE - hw::ledger::log_hexbuffer("Prehash AKV input", (char*)&this->buffer_recv[64], 3*32); + log_hexbuffer("Prehash AKV input", (char*)&this->buffer_recv[64], 3*32); #endif } @@ -1377,7 +1566,6 @@ namespace hw { bool device_ledger::mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &II) { AUTO_LOCK_CMD(); - unsigned char options; #ifdef DEBUG_HWDEVICE const rct::key H_x = H; @@ -1421,7 +1609,6 @@ namespace hw { bool device_ledger::mlsag_prepare(rct::key &a, rct::key &aG) { AUTO_LOCK_CMD(); - unsigned char options; #ifdef DEBUG_HWDEVICE rct::key a_x; @@ -1444,7 +1631,6 @@ namespace hw { bool device_ledger::mlsag_hash(const rct::keyV &long_message, rct::key &c) { AUTO_LOCK_CMD(); - unsigned char options; size_t cnt; #ifdef DEBUG_HWDEVICE diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index 2f5beb044..252354e1c 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -56,12 +56,14 @@ namespace hw { rct::key Aout; rct::key Bout; bool is_subaddress; + bool is_change_address; + bool additional_key ; size_t index; rct::key Pout; rct::key AKout; - ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, size_t index, const rct::key& P,const rct::key& AK); + ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, bool is_subaddress, bool is_change_address, size_t index, const rct::key& P,const rct::key& AK); ABPkeys(const ABPkeys& keys) ; - ABPkeys() {index=0;is_subaddress=false;} + ABPkeys() {index=0;is_subaddress=false;is_subaddress=false;is_change_address=false;} }; class Keymap { @@ -85,7 +87,6 @@ namespace hw { //IO hw::io::device_io_hid hw_device; - std::string full_name; unsigned int length_send; unsigned char buffer_send[BUFFER_SEND_SIZE]; unsigned int length_recv; @@ -95,6 +96,7 @@ namespace hw { void logCMD(void); void logRESP(void); unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); + unsigned int exchange_wait_on_input(unsigned int ok=0x9000, unsigned int mask=0xFFFF); void reset_buffer(void); int set_command_header(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); int set_command_header_noopt(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); @@ -105,7 +107,9 @@ namespace hw { device_mode mode; // map public destination key to ephemeral destination key Keymap key_map; - + bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const bool is_change, + const bool need_additional, const size_t real_output_index, + const rct::key &amount_key, const crypto::public_key &out_eph_public_key); // To speed up blockchain parsing the view key maybe handle here. crypto::secret_key viewkey; bool has_view_key; @@ -191,12 +195,15 @@ namespace hw { bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; - - bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, - const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_format) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_format) override; + bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub, const crypto::secret_key &tx_key, + const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index, + const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys, + std::vector<crypto::public_key> &additional_tx_public_keys, + std::vector<rct::key> &amount_keys, + crypto::public_key &out_eph_public_key) override; bool mlsag_prehash(const std::string &blob, size_t inputs_size, size_t outputs_size, const rct::keyV &hashes, const rct::ctkeyV &outPk, rct::key &prehash) override; bool mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &rvII) override; diff --git a/src/device/log.cpp b/src/device/log.cpp index c9d3b551b..616ad8e90 100644 --- a/src/device/log.cpp +++ b/src/device/log.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -66,7 +66,7 @@ namespace hw { void decrypt(char* buf, size_t len) { - #ifdef IODUMMYCRYPT_HWDEVICE + #if defined(IODUMMYCRYPT_HWDEVICE) || defined(IONOCRYPT_HWDEVICE) size_t i; if (len == 32) { //view key? @@ -86,11 +86,13 @@ namespace hw { return; } } + #if defined(IODUMMYCRYPT_HWDEVICE) //std decrypt: XOR.55h for (i = 0; i<len;i++) { buf[i] ^= 0x55; } #endif + #endif } crypto::key_derivation decrypt(const crypto::key_derivation &derivation) { diff --git a/src/device/log.hpp b/src/device/log.hpp index 25a214a6c..fb7ba1fb0 100644 --- a/src/device/log.hpp +++ b/src/device/log.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt index 7f979389a..250939da7 100644 --- a/src/device_trezor/CMakeLists.txt +++ b/src/device_trezor/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2017, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -67,6 +67,12 @@ set(trezor_private_headers) if(DEVICE_TREZOR_READY) message(STATUS "Trezor support enabled") + if(USE_DEVICE_TREZOR_DEBUG) + list(APPEND trezor_headers trezor/debug_link.hpp trezor/messages/messages-debug.pb.h) + list(APPEND trezor_sources trezor/debug_link.cpp trezor/messages/messages-debug.pb.cc) + message(STATUS "Trezor debugging enabled") + endif() + monero_private_headers(device_trezor ${device_private_headers}) diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 8868fb995..ceb6111e0 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -97,7 +97,14 @@ namespace trezor { auto res = get_view_key(); CHECK_AND_ASSERT_MES(res->watch_key().size() == 32, false, "Trezor returned invalid view key"); + // Trezor does not make use of spendkey of the device API. + // Ledger loads encrypted spendkey, Trezor loads null key (never leaves device). + // In the test (debugging mode) we need to leave this field intact as it is already set by + // the debugging code and need to remain same for the testing purposes. +#ifndef WITH_TREZOR_DEBUGGING spendkey = crypto::null_skey; // not given +#endif + memcpy(viewkey.data, res->watch_key().data(), 32); return true; @@ -362,13 +369,9 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized"); // make sure the caller did not reset features const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0; - if (nonce_required){ + if (nonce_required && init_msg->tsx_data().payment_id().size() == 8){ // Versions 2.0.9 and lower do not support payment ID - CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information"); - const uint32_t vma = m_features->major_version(); - const uint32_t vmi = m_features->minor_version(); - const uint32_t vpa = m_features->patch_version(); - if (vma < 2 || (vma == 2 && vmi == 0 && vpa <= 9)) { + if (get_version() <= pack_version(2, 0, 9)) { throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update."); } } @@ -393,7 +396,7 @@ namespace trezor { const bool nonce_required = tdata.tsx_data.has_payment_id() && tdata.tsx_data.payment_id().size() > 0; const bool has_nonce = cryptonote::find_tx_extra_field_by_type(tx_extra_fields, nonce); - CHECK_AND_ASSERT_THROW_MES(has_nonce == nonce_required, "Transaction nonce presence inconsistent"); + CHECK_AND_ASSERT_THROW_MES(has_nonce || !nonce_required, "Transaction nonce not present"); if (nonce_required){ const std::string & payment_id = tdata.tsx_data.payment_id(); diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index 1f08be887..75f6d5875 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 5071932ee..1892b47d9 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -44,7 +44,9 @@ namespace trezor { const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080}; device_trezor_base::device_trezor_base(): m_callback(nullptr) { - +#ifdef WITH_TREZOR_DEBUGGING + m_debug = false; +#endif } device_trezor_base::~device_trezor_base() { @@ -130,6 +132,10 @@ namespace trezor { } m_transport->open(); + +#ifdef WITH_TREZOR_DEBUGGING + setup_debug(); +#endif return true; } catch(std::exception const& e){ @@ -153,6 +159,13 @@ namespace trezor { return false; } } + +#ifdef WITH_TREZOR_DEBUGGING + if (m_debug_callback) { + m_debug_callback->on_disconnect(); + m_debug_callback = nullptr; + } +#endif return true; } @@ -355,6 +368,32 @@ namespace trezor { device_state_reset_unsafe(); } +#ifdef WITH_TREZOR_DEBUGGING +#define TREZOR_CALLBACK(method, ...) do { \ + if (m_debug_callback) m_debug_callback->method(__VA_ARGS__); \ + if (m_callback) m_callback->method(__VA_ARGS__); \ +}while(0) + + void device_trezor_base::setup_debug(){ + if (!m_debug){ + return; + } + + if (!m_debug_callback){ + CHECK_AND_ASSERT_THROW_MES(m_transport, "Transport does not exist"); + auto debug_transport = m_transport->find_debug(); + if (debug_transport) { + m_debug_callback = std::make_shared<trezor_debug_callback>(debug_transport); + } else { + MDEBUG("Transport does not have debug link option"); + } + } + } + +#else +#define TREZOR_CALLBACK(method, ...) do { if (m_callback) m_callback->method(__VA_ARGS__); } while(0) +#endif + void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) { CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); @@ -363,10 +402,7 @@ namespace trezor { messages::common::ButtonAck ack; write_raw(&ack); - if (m_callback){ - m_callback->on_button_request(); - } - + TREZOR_CALLBACK(on_button_request); resp = read_raw(); } @@ -377,9 +413,7 @@ namespace trezor { epee::wipeable_string pin; - if (m_callback){ - m_callback->on_pin_request(pin); - } + TREZOR_CALLBACK(on_pin_request, pin); // TODO: remove PIN from memory messages::common::PinMatrixAck m; @@ -393,9 +427,7 @@ namespace trezor { MDEBUG("on_passhprase_request, on device: " << msg->on_device()); epee::wipeable_string passphrase; - if (m_callback){ - m_callback->on_passphrase_request(msg->on_device(), passphrase); - } + TREZOR_CALLBACK(on_passphrase_request, msg->on_device(), passphrase); messages::common::PassphraseAck m; if (!msg->on_device()){ @@ -421,5 +453,67 @@ namespace trezor { resp = call_raw(&m); } +#ifdef WITH_TREZOR_DEBUGGING + void device_trezor_base::wipe_device() + { + auto msg = std::make_shared<messages::management::WipeDevice>(); + auto ret = client_exchange<messages::common::Success>(msg); + (void)ret; + init_device(); + } + + void device_trezor_base::init_device() + { + auto msg = std::make_shared<messages::management::Initialize>(); + m_features = client_exchange<messages::management::Features>(msg); + } + + void device_trezor_base::load_device(const std::string & mnemonic, const std::string & pin, + bool passphrase_protection, const std::string & label, const std::string & language, + bool skip_checksum, bool expand) + { + if (m_features && m_features->initialized()){ + throw std::runtime_error("Device is initialized already. Call device.wipe() and try again."); + } + + auto msg = std::make_shared<messages::management::LoadDevice>(); + msg->set_mnemonic(mnemonic); + msg->set_pin(pin); + msg->set_passphrase_protection(passphrase_protection); + msg->set_label(label); + msg->set_language(language); + msg->set_skip_checksum(skip_checksum); + auto ret = client_exchange<messages::common::Success>(msg); + (void)ret; + + init_device(); + } + + trezor_debug_callback::trezor_debug_callback(std::shared_ptr<Transport> & debug_transport){ + m_debug_link = std::make_shared<DebugLink>(); + m_debug_link->init(debug_transport); + } + + void trezor_debug_callback::on_button_request() { + if (m_debug_link) m_debug_link->press_yes(); + } + + void trezor_debug_callback::on_pin_request(epee::wipeable_string &pin) { + + } + + void trezor_debug_callback::on_passphrase_request(bool on_device, epee::wipeable_string &passphrase) { + + } + + void trezor_debug_callback::on_passphrase_state_request(const std::string &state) { + + } + + void trezor_debug_callback::on_disconnect(){ + if (m_debug_link) m_debug_link->close(); + } +#endif + #endif //WITH_DEVICE_TREZOR }} diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 3c35e8aca..8942df797 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -42,6 +42,10 @@ #include "cryptonote_config.h" #include "trezor.hpp" +#ifdef WITH_TREZOR_DEBUGGING +#include "trezor/debug_link.hpp" +#endif + //automatic lock one more level on device ensuring the current thread is allowed to use it #define AUTO_LOCK_CMD() \ /* lock both mutexes without deadlock*/ \ @@ -57,6 +61,23 @@ namespace trezor { #ifdef WITH_DEVICE_TREZOR class device_trezor_base; +#ifdef WITH_TREZOR_DEBUGGING + class trezor_debug_callback { + public: + trezor_debug_callback()=default; + explicit trezor_debug_callback(std::shared_ptr<Transport> & debug_transport); + + void on_button_request(); + void on_pin_request(epee::wipeable_string &pin); + void on_passphrase_request(bool on_device, epee::wipeable_string &passphrase); + void on_passphrase_state_request(const std::string &state); + void on_disconnect(); + protected: + std::shared_ptr<DebugLink> m_debug_link; + }; + +#endif + /** * TREZOR device template with basic functions */ @@ -77,6 +98,13 @@ namespace trezor { cryptonote::network_type network_type; +#ifdef WITH_TREZOR_DEBUGGING + std::shared_ptr<trezor_debug_callback> m_debug_callback; + bool m_debug; + + void setup_debug(); +#endif + // // Internal methods // @@ -103,7 +131,7 @@ namespace trezor { * @throws UnexpectedMessageException if the response message type is different than expected. * Exception contains message type and the message itself. */ - template<class t_message> + template<class t_message=google::protobuf::Message> std::shared_ptr<t_message> client_exchange(const std::shared_ptr<const google::protobuf::Message> &req, const boost::optional<messages::MessageType> & resp_type = boost::none, @@ -229,6 +257,12 @@ namespace trezor { return m_features; } + uint64_t get_version() const { + CHECK_AND_ASSERT_THROW_MES(m_features, "Features not loaded"); + CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information"); + return pack_version(m_features->major_version(), m_features->minor_version(), m_features->patch_version()); + } + void set_derivation_path(const std::string &deriv_path) override; /* ======================================================================= */ @@ -268,6 +302,23 @@ namespace trezor { void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg); void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg); void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg); + +#ifdef WITH_TREZOR_DEBUGGING + void set_debug(bool debug){ + m_debug = debug; + } + + void set_debug_callback(std::shared_ptr<trezor_debug_callback> & debug_callback){ + m_debug_callback = debug_callback; + } + + void wipe_device(); + void init_device(); + void load_device(const std::string & mnemonic, const std::string & pin="", bool passphrase_protection=false, + const std::string & label="test", const std::string & language="english", + bool skip_checksum=false, bool expand=false); + +#endif }; #endif diff --git a/src/device_trezor/trezor.hpp b/src/device_trezor/trezor.hpp index 97dc0a957..396a27534 100644 --- a/src/device_trezor/trezor.hpp +++ b/src/device_trezor/trezor.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/debug_link.cpp b/src/device_trezor/trezor/debug_link.cpp new file mode 100644 index 000000000..c7ee59afe --- /dev/null +++ b/src/device_trezor/trezor/debug_link.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "debug_link.hpp" + +namespace hw{ +namespace trezor{ + + DebugLink::DebugLink(){ + + } + + DebugLink::~DebugLink(){ + if (m_transport){ + close(); + } + } + + void DebugLink::init(std::shared_ptr<Transport> & transport){ + CHECK_AND_ASSERT_THROW_MES(!m_transport, "Already initialized"); + m_transport = transport; + m_transport->open(); + } + + void DebugLink::close(){ + CHECK_AND_ASSERT_THROW_MES(m_transport, "Not initialized"); + if (m_transport) m_transport->close(); + } + + std::shared_ptr<messages::debug::DebugLinkState> DebugLink::state(){ + return call<messages::debug::DebugLinkState>( + messages::debug::DebugLinkGetState(), + boost::make_optional(messages::MessageType_DebugLinkGetState)); + } + + void DebugLink::input_word(const std::string & word){ + messages::debug::DebugLinkDecision decision; + decision.set_input(word); + call(decision, boost::none, true); + } + + void DebugLink::input_button(bool button){ + messages::debug::DebugLinkDecision decision; + decision.set_yes_no(button); + call(decision, boost::none, true); + } + + void DebugLink::input_swipe(bool swipe){ + messages::debug::DebugLinkDecision decision; + decision.set_up_down(swipe); + call(decision, boost::none, true); + } + + void DebugLink::stop(){ + messages::debug::DebugLinkStop msg; + call(msg, boost::none, true); + } + + + + + +} +}
\ No newline at end of file diff --git a/src/device_trezor/trezor/debug_link.hpp b/src/device_trezor/trezor/debug_link.hpp new file mode 100644 index 000000000..adf5f1d8f --- /dev/null +++ b/src/device_trezor/trezor/debug_link.hpp @@ -0,0 +1,93 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEBUG_LINK_H +#define MONERO_DEBUG_LINK_H + +#include "transport.hpp" +#include "messages/messages-debug.pb.h" + + +namespace hw { +namespace trezor { + + class DebugLink { + public: + + DebugLink(); + virtual ~DebugLink(); + + void init(std::shared_ptr<Transport> & transport); + void close(); + + std::shared_ptr<messages::debug::DebugLinkState> state(); + void input_word(const std::string & word); + void input_button(bool button); + void input_swipe(bool swipe); + void press_yes() { input_button(true); } + void press_no() { input_button(false); } + void stop(); + + template<class t_message=messages::debug::DebugLinkState> + std::shared_ptr<t_message> call( + const google::protobuf::Message & req, + const boost::optional<messages::MessageType> &resp_type = boost::none, + bool no_wait = false) + { + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + + m_transport->write(req); + if (no_wait){ + return nullptr; + } + + // Read the response + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + m_transport->read(msg_resp, &msg_resp_type); + + messages::MessageType required_type = resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>(); + if (msg_resp_type == required_type) { + return message_ptr_retype<t_message>(msg_resp); + } else if (msg_resp_type == messages::MessageType_Failure){ + throw_failure_exception(dynamic_cast<messages::common::Failure*>(msg_resp.get())); + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + }; + + private: + std::shared_ptr<Transport> m_transport; + + }; + +} +} + +#endif //MONERO_DEBUG_LINK_H diff --git a/src/device_trezor/trezor/exceptions.hpp b/src/device_trezor/trezor/exceptions.hpp index 197dc43a4..41e8c2875 100644 --- a/src/device_trezor/trezor/exceptions.hpp +++ b/src/device_trezor/trezor/exceptions.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/device_trezor/trezor/messages_map.cpp b/src/device_trezor/trezor/messages_map.cpp index b0d1aa254..6956b369a 100644 --- a/src/device_trezor/trezor/messages_map.cpp +++ b/src/device_trezor/trezor/messages_map.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -33,6 +33,10 @@ #include "messages/messages-management.pb.h" #include "messages/messages-monero.pb.h" +#ifdef WITH_TREZOR_DEBUGGING +#include "messages/messages-debug.pb.h" +#endif + using namespace std; using namespace hw::trezor; @@ -45,6 +49,9 @@ namespace trezor "hw.trezor.messages.", "hw.trezor.messages.common.", "hw.trezor.messages.management.", +#ifdef WITH_TREZOR_DEBUGGING + "hw.trezor.messages.debug.", +#endif "hw.trezor.messages.monero." }; @@ -68,6 +75,10 @@ namespace trezor hw::trezor::messages::management::Cancel::default_instance(); hw::trezor::messages::monero::MoneroGetAddress::default_instance(); +#ifdef WITH_TREZOR_DEBUGGING + hw::trezor::messages::debug::DebugLinkDecision::default_instance(); +#endif + google::protobuf::Descriptor const * desc = nullptr; for(const string &text : PACKAGES){ desc = google::protobuf::DescriptorPool::generated_pool() diff --git a/src/device_trezor/trezor/messages_map.hpp b/src/device_trezor/trezor/messages_map.hpp index f61338f09..28fe9f4c2 100644 --- a/src/device_trezor/trezor/messages_map.hpp +++ b/src/device_trezor/trezor/messages_map.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -62,14 +62,14 @@ namespace trezor { static messages::MessageType get_message_wire_number(const google::protobuf::Message & msg); static messages::MessageType get_message_wire_number(const std::string & msg_name); - template<class t_message> + template<class t_message=google::protobuf::Message> static messages::MessageType get_message_wire_number() { BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); return get_message_wire_number(t_message::default_instance().GetDescriptor()->name()); } }; - template<class t_message> + template<class t_message=google::protobuf::Message> std::shared_ptr<t_message> message_ptr_retype(std::shared_ptr<google::protobuf::Message> & in){ BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); if (!in){ @@ -79,7 +79,7 @@ namespace trezor { return std::dynamic_pointer_cast<t_message>(in); } - template<class t_message> + template<class t_message=google::protobuf::Message> std::shared_ptr<t_message> message_ptr_retype_static(std::shared_ptr<google::protobuf::Message> & in){ BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); if (!in){ diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index c4a92426c..79f6adac9 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -877,6 +877,9 @@ namespace tx { valueS.SetString(m_ct.enc_salt2.c_str(), m_ct.enc_salt2.size()); json.AddMember("salt2", valueS, json.GetAllocator()); + valueS.SetString(m_ct.tx_prefix_hash.c_str(), m_ct.tx_prefix_hash.size()); + json.AddMember("tx_prefix_hash", valueS, json.GetAllocator()); + valueS.SetString(m_ct.enc_keys.c_str(), m_ct.enc_keys.size()); json.AddMember("enc_keys", valueS, json.GetAllocator()); diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index 99211efed..23f94f948 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -270,7 +270,7 @@ namespace tx { throw std::invalid_argument("RV not initialized"); } auto tp = m_ct.rv->type; - return tp == rct::RCTTypeBulletproof; + return tp == rct::RCTTypeBulletproof || tp == rct::RCTTypeBulletproof2; } bool is_offloading() const { diff --git a/src/device_trezor/trezor/tools/build_protob.py b/src/device_trezor/trezor/tools/build_protob.py index 2611f3296..eb32f6b4d 100644 --- a/src/device_trezor/trezor/tools/build_protob.py +++ b/src/device_trezor/trezor/tools/build_protob.py @@ -2,6 +2,12 @@ import os import subprocess import sys +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument("-d", "--debug-msg", default=False, action="store_const", const=True, help="Build debug messages") +args = parser.parse_args() CWD = os.path.dirname(os.path.realpath(__file__)) ROOT_DIR = os.path.abspath(os.path.join(CWD, "..", "..", "..", "..")) @@ -24,6 +30,10 @@ try: "messages-management.proto", "messages-monero.proto", ] + + if args.debug_msg: + selected += ["messages-debug.proto"] + proto_srcs = [os.path.join(TREZOR_COMMON, "protob", x) for x in selected] exec_args = [ sys.executable, diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index cd66e59e8..991ba3395 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -84,6 +84,17 @@ namespace trezor{ return std::string(in.GetString()); } + uint64_t pack_version(uint32_t major, uint32_t minor, uint32_t patch) + { + // packing (major, minor, patch) to 64 B: 16 B | 24 B | 24 B + const unsigned bits_1 = 16; + const unsigned bits_2 = 24; + const uint32_t mask_1 = (1 << bits_1) - 1; + const uint32_t mask_2 = (1 << bits_2) - 1; + CHECK_AND_ASSERT_THROW_MES(major <= mask_1 && minor <= mask_2 && patch <= mask_2, "Version numbers overflow packing scheme"); + return patch | (((uint64_t)minor) << bits_2) | (((uint64_t)major) << (bits_1 + bits_2)); + } + // // Helpers // @@ -212,6 +223,40 @@ namespace trezor{ msg = msg_wrap; } + Transport::Transport(): m_open_counter(0) { + + } + + bool Transport::pre_open(){ + if (m_open_counter > 0){ + MTRACE("Already opened, count: " << m_open_counter); + m_open_counter += 1; + return false; + + } else if (m_open_counter < 0){ + MTRACE("Negative open value: " << m_open_counter); + + } + + // Caller should set m_open_counter to 1 after open + m_open_counter = 0; + return true; + } + + bool Transport::pre_close(){ + m_open_counter -= 1; + + if (m_open_counter < 0){ + MDEBUG("Already closed. Counter " << m_open_counter); + + } else if (m_open_counter == 0) { + return true; + + } + + return false; + } + // // Bridge transport // @@ -246,6 +291,10 @@ namespace trezor{ } void BridgeTransport::open() { + if (!pre_open()){ + return; + } + if (!m_device_path){ throw exc::CommunicationException("Coud not open, empty device path"); } @@ -259,9 +308,15 @@ namespace trezor{ } m_session = boost::make_optional(json_get_string(bridge_res["session"])); + m_open_counter = 1; } void BridgeTransport::close() { + if (!pre_close()){ + return; + } + + MTRACE("Closing Trezor:BridgeTransport"); if (!m_device_path || !m_session){ throw exc::CommunicationException("Device not open"); } @@ -423,6 +478,10 @@ namespace trezor{ } void UdpTransport::open() { + if (!pre_open()){ + return; + } + udp::resolver resolver(m_io_service); udp::resolver::query query(udp::v4(), m_device_host, std::to_string(m_device_port)); m_endpoint = *resolver.resolve(query); @@ -434,10 +493,16 @@ namespace trezor{ check_deadline(); m_proto->session_begin(*this); + m_open_counter = 1; } void UdpTransport::close() { - if (!m_socket){ + if (!pre_close()){ + return; + } + + MTRACE("Closing Trezor:UdpTransport"); + if (!m_socket) { throw exc::CommunicationException("Socket is already closed"); } @@ -446,6 +511,19 @@ namespace trezor{ m_socket = nullptr; } + std::shared_ptr<Transport> UdpTransport::find_debug() { +#ifdef WITH_TREZOR_DEBUGGING + std::shared_ptr<UdpTransport> t = std::make_shared<UdpTransport>(); + t->m_proto = std::make_shared<ProtocolV1>(); + t->m_device_host = m_device_host; + t->m_device_port = m_device_port + 1; + return t; +#else + MINFO("Debug link is disabled in production"); + return nullptr; +#endif + } + void UdpTransport::write_chunk(const void * buff, size_t size){ require_socket(); @@ -660,8 +738,7 @@ namespace trezor{ WebUsbTransport::WebUsbTransport( boost::optional<libusb_device_descriptor*> descriptor, boost::optional<std::shared_ptr<Protocol>> proto - ): m_conn_count(0), - m_usb_session(nullptr), m_usb_device(nullptr), m_usb_device_handle(nullptr), + ): m_usb_session(nullptr), m_usb_device(nullptr), m_usb_device_handle(nullptr), m_bus_id(-1), m_device_addr(-1) { if (descriptor){ @@ -672,7 +749,7 @@ namespace trezor{ m_proto = proto ? proto.get() : std::make_shared<ProtocolV1>(); -#ifdef WITH_TREZOR_DEBUG +#ifdef WITH_TREZOR_DEBUGGING m_debug_mode = false; #endif } @@ -757,12 +834,10 @@ namespace trezor{ }; void WebUsbTransport::open() { - const int interface = get_interface(); - if (m_conn_count > 0){ - MTRACE("Already opened, count: " << m_conn_count); - m_conn_count += 1; + if (!pre_open()){ return; } + const int interface = get_interface(); #define TREZOR_DESTROY_SESSION() do { libusb_exit(m_usb_session); m_usb_session = nullptr; } while(0) @@ -840,45 +915,55 @@ namespace trezor{ throw exc::DeviceAcquireException("Unable to claim libusb device"); } - m_conn_count = 1; + m_open_counter = 1; m_proto->session_begin(*this); #undef TREZOR_DESTROY_SESSION }; void WebUsbTransport::close() { - m_conn_count -= 1; - - if (m_conn_count < 0){ - MERROR("Close counter is negative: " << m_conn_count); - - } else if (m_conn_count == 0){ - MTRACE("Closing webusb device"); + if (!pre_close()){ + return; + } - m_proto->session_end(*this); + MTRACE("Closing Trezor:WebUsbTransport"); + m_proto->session_end(*this); - int r = libusb_release_interface(m_usb_device_handle, get_interface()); - if (r != 0){ - MERROR("Could not release libusb interface: " << r); - } + int r = libusb_release_interface(m_usb_device_handle, get_interface()); + if (r != 0){ + MERROR("Could not release libusb interface: " << r); + } - m_usb_device = nullptr; - if (m_usb_device_handle) { - libusb_close(m_usb_device_handle); - m_usb_device_handle = nullptr; - } + m_usb_device = nullptr; + if (m_usb_device_handle) { + libusb_close(m_usb_device_handle); + m_usb_device_handle = nullptr; + } - if (m_usb_session) { - libusb_exit(m_usb_session); - m_usb_session = nullptr; - } + if (m_usb_session) { + libusb_exit(m_usb_session); + m_usb_session = nullptr; } }; + std::shared_ptr<Transport> WebUsbTransport::find_debug() { +#ifdef WITH_TREZOR_DEBUGGING + require_device(); + auto t = std::make_shared<WebUsbTransport>(boost::make_optional(m_usb_device_desc.get())); + t->m_bus_id = m_bus_id; + t->m_device_addr = m_device_addr; + t->m_port_numbers = m_port_numbers; + t->m_debug_mode = true; + return t; +#else + MINFO("Debug link is disabled in production"); + return nullptr; +#endif + } int WebUsbTransport::get_interface() const{ const int INTERFACE_NORMAL = 0; -#ifdef WITH_TREZOR_DEBUG +#ifdef WITH_TREZOR_DEBUGGING const int INTERFACE_DEBUG = 1; return m_debug_mode ? INTERFACE_DEBUG : INTERFACE_NORMAL; #else @@ -888,7 +973,7 @@ namespace trezor{ unsigned char WebUsbTransport::get_endpoint() const{ const unsigned char ENDPOINT_NORMAL = 1; -#ifdef WITH_TREZOR_DEBUG +#ifdef WITH_TREZOR_DEBUGGING const unsigned char ENDPOINT_DEBUG = 2; return m_debug_mode ? ENDPOINT_DEBUG : ENDPOINT_NORMAL; #else @@ -1047,4 +1132,3 @@ namespace trezor{ } } - diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp index 50c31cf73..2945b3184 100644 --- a/src/device_trezor/trezor/transport.hpp +++ b/src/device_trezor/trezor/transport.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -62,6 +62,8 @@ namespace trezor { const std::string DEFAULT_BRIDGE = "127.0.0.1:21325"; + uint64_t pack_version(uint32_t major, uint32_t minor=0, uint32_t patch=0); + // Base HTTP comm serialization. bool t_serialize(const std::string & in, std::string & out); bool t_serialize(const json_val & in, std::string & out); @@ -134,7 +136,7 @@ namespace trezor { class Transport { public: - Transport() = default; + Transport(); virtual ~Transport() = default; virtual bool ping() { return false; }; @@ -144,10 +146,16 @@ namespace trezor { virtual void close(){}; virtual void write(const google::protobuf::Message & req) =0; virtual void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) =0; + virtual std::shared_ptr<Transport> find_debug() { return nullptr; }; virtual void write_chunk(const void * buff, size_t size) { }; virtual size_t read_chunk(void * buff, size_t size) { return 0; }; virtual std::ostream& dump(std::ostream& o) const { return o << "Transport<>"; } + protected: + long m_open_counter; + + virtual bool pre_open(); + virtual bool pre_close(); }; // Bridge transport @@ -162,7 +170,7 @@ namespace trezor { m_session(boost::none), m_device_info(boost::none) { - m_http_client.set_server(m_bridge_host, boost::none, false); + m_http_client.set_server(m_bridge_host, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_disabled); } virtual ~BridgeTransport() = default; @@ -212,6 +220,7 @@ namespace trezor { void open() override; void close() override; + std::shared_ptr<Transport> find_debug() override; void write(const google::protobuf::Message &req) override; void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; @@ -259,6 +268,7 @@ namespace trezor { void open() override; void close() override; + std::shared_ptr<Transport> find_debug() override; void write(const google::protobuf::Message &req) override; void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; @@ -274,7 +284,6 @@ namespace trezor { int get_interface() const; unsigned char get_endpoint() const; - int m_conn_count; std::shared_ptr<Protocol> m_proto; libusb_context *m_usb_session; @@ -285,7 +294,7 @@ namespace trezor { int m_bus_id; int m_device_addr; -#ifdef WITH_TREZOR_DEBUG +#ifdef WITH_TREZOR_DEBUGGING bool m_debug_mode; #endif }; @@ -309,7 +318,7 @@ namespace trezor { /** * Transforms path to the particular transport */ - template<class t_transport> + template<class t_transport=Transport> std::shared_ptr<t_transport> transport_typed(const std::string & path){ auto t = transport(path); if (!t){ @@ -362,7 +371,7 @@ namespace trezor { * @throws UnexpectedMessageException if the response message type is different than expected. * Exception contains message type and the message itself. */ - template<class t_message> + template<class t_message=google::protobuf::Message> std::shared_ptr<t_message> exchange_message(Transport & transport, const google::protobuf::Message & req, boost::optional<messages::MessageType> resp_type = boost::none) diff --git a/src/device_trezor/trezor/trezor_defs.hpp b/src/device_trezor/trezor/trezor_defs.hpp index 30e76eadc..f0697cdb5 100644 --- a/src/device_trezor/trezor/trezor_defs.hpp +++ b/src/device_trezor/trezor/trezor_defs.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/gen_multisig/CMakeLists.txt b/src/gen_multisig/CMakeLists.txt index 18a6a9efe..3a5e29273 100644 --- a/src/gen_multisig/CMakeLists.txt +++ b/src/gen_multisig/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2018, The Monero Project +# Copyright (c) 2017-2019, The Monero Project # # All rights reserved. # diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index 8262e86f7..06019c50e 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt new file mode 100644 index 000000000..1f369f114 --- /dev/null +++ b/src/lmdb/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2014-2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(lmdb_sources database.cpp error.cpp table.cpp value_stream.cpp) +set(lmdb_headers database.h error.h key_stream.h table.h transaction.h util.h value_stream.h) + +monero_add_library(lmdb_lib ${lmdb_sources} ${lmdb_headers}) +target_link_libraries(lmdb_lib common ${LMDB_LIBRARY}) diff --git a/src/lmdb/database.cpp b/src/lmdb/database.cpp new file mode 100644 index 000000000..c6b244671 --- /dev/null +++ b/src/lmdb/database.cpp @@ -0,0 +1,187 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "database.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +#ifdef _WIN32 +namespace +{ + constexpr const mdb_mode_t open_flags = 0; +} +#else +#include <sys/stat.h> + +namespace +{ + constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR); +} +#endif + +namespace lmdb +{ + namespace + { + constexpr const std::size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB + void acquire_context(context& ctx) noexcept + { + while (ctx.lock.test_and_set()); + ++(ctx.active); + ctx.lock.clear(); + } + + void release_context(context& ctx) noexcept + { + --(ctx.active); + } + } + + void release_read_txn::operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + { + MDB_env* const env = mdb_txn_env(ptr); + abort_txn{}(ptr); + if (env) + { + context* ctx = reinterpret_cast<context*>(mdb_env_get_userctx(env)); + if (ctx) + release_context(*ctx); + } + } + } + + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept + { + MONERO_PRECOND(path != nullptr); + + MDB_env* obj = nullptr; + MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj))); + environment out{obj}; + + MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs)); + MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags)); + return {std::move(out)}; + } + + expect<write_txn> database::do_create_txn(unsigned int flags) noexcept + { + MONERO_PRECOND(handle() != nullptr); + + for (unsigned attempts = 0; attempts < 3; ++attempts) + { + acquire_context(ctx); + + MDB_txn* txn = nullptr; + const int err = + mdb_txn_begin(handle(), nullptr, flags, &txn); + if (!err && txn != nullptr) + return write_txn{txn}; + + release_context(ctx); + if (err != MDB_MAP_RESIZED) + return {lmdb::error(err)}; + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_RESIZED)}; + } + + database::database(environment env) + : env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT} + { + if (handle()) + { + const int err = mdb_env_set_userctx(handle(), std::addressof(ctx)); + if (err) + MONERO_THROW(lmdb::error(err), "Failed to set user context"); + } + } + + database::~database() noexcept + { + while (ctx.active); + } + + expect<void> database::resize() noexcept + { + MONERO_PRECOND(handle() != nullptr); + + while (ctx.lock.test_and_set()); + while (ctx.active); + + MDB_envinfo info{}; + MONERO_LMDB_CHECK(mdb_env_info(handle(), &info)); + + const std::size_t resize = std::min(info.me_mapsize, max_resize); + const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize); + ctx.lock.clear(); + if (err) + return {lmdb::error(err)}; + return success(); + } + + expect<read_txn> database::create_read_txn(suspended_txn txn) noexcept + { + if (txn) + { + acquire_context(ctx); + const int err = mdb_txn_renew(txn.get()); + if (err) + { + release_context(ctx); + return {lmdb::error(err)}; + } + return read_txn{txn.release()}; + } + auto new_txn = do_create_txn(MDB_RDONLY); + if (new_txn) + return read_txn{new_txn->release()}; + return new_txn.error(); + } + + expect<suspended_txn> database::reset_txn(read_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + mdb_txn_reset(txn.get()); + release_context(ctx); + return suspended_txn{txn.release()}; + } + + expect<write_txn> database::create_write_txn() noexcept + { + return do_create_txn(0); + } + + expect<void> database::commit(write_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + MONERO_LMDB_CHECK(mdb_txn_commit(txn.get())); + txn.release(); + release_context(ctx); + return success(); + } +} // lmdb diff --git a/src/lmdb/database.h b/src/lmdb/database.h new file mode 100644 index 000000000..269f8c8a1 --- /dev/null +++ b/src/lmdb/database.h @@ -0,0 +1,138 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <atomic> +#include <cstddef> +#include <lmdb.h> +#include <memory> +#include <type_traits> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/transaction.h" + +namespace lmdb +{ + //! Closes LMDB environment handle. + struct close_env + { + void operator()(MDB_env* ptr) const noexcept + { + if (ptr) + mdb_env_close(ptr); + } + }; + + using environment = std::unique_ptr<MDB_env, close_env>; + + //! \return LMDB environment at `path` with a max of `max_dbs` tables. + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept; + + //! Context given to LMDB. + struct context + { + std::atomic<std::size_t> active; + std::atomic_flag lock; + }; + + //! Manages a LMDB environment for safe memory-map resizing. Thread-safe. + class database + { + environment env; + context ctx; + + //! \return The LMDB environment associated with the object. + MDB_env* handle() const noexcept { return env.get(); } + + expect<write_txn> do_create_txn(unsigned int flags) noexcept; + + public: + database(environment env); + + database(database&&) = delete; + database(database const&) = delete; + + virtual ~database() noexcept; + + database& operator=(database&&) = delete; + database& operator=(database const&) = delete; + + /*! + Resize the memory map for the LMDB environment. Will block until + all reads/writes on the environment complete. + */ + expect<void> resize() noexcept; + + //! \return A read only LMDB transaction, reusing `txn` if provided. + expect<read_txn> create_read_txn(suspended_txn txn = nullptr) noexcept; + + //! \return `txn` after releasing context. + expect<suspended_txn> reset_txn(read_txn txn) noexcept; + + //! \return A read-write LMDB transaction. + expect<write_txn> create_write_txn() noexcept; + + //! Commit the read-write transaction. + expect<void> commit(write_txn txn) noexcept; + + /*! + Create a write transaction, pass it to `f`, then try to commit + the write if `f` succeeds. + + \tparam F must be callable with signature `expect<T>(MDB_txn&)`. + \param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`. + + \return The result of calling `f`. + */ + template<typename F> + typename std::result_of<F(MDB_txn&)>::type try_write(F f, unsigned attempts = 3) + { + for (unsigned i = 0; i < attempts; ++i) + { + expect<write_txn> txn = create_write_txn(); + if (!txn) + return txn.error(); + + MONERO_PRECOND(*txn != nullptr); + const auto wrote = f(*(*txn)); + if (wrote) + { + MONERO_CHECK(commit(std::move(*txn))); + return wrote; + } + if (wrote != lmdb::error(MDB_MAP_FULL)) + return wrote; + + txn->reset(); + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_FULL)}; + } + }; +} // lmdb + diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp new file mode 100644 index 000000000..359677064 --- /dev/null +++ b/src/lmdb/error.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +#include <lmdb.h> +#include <string> + +namespace { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "lmdb::error_category()"; + } + + virtual std::string message(int value) const override final + { + char const* const msg = mdb_strerror(value); + if (msg) + return msg; + return "Unknown lmdb::error_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + switch (value) + { + case MDB_KEYEXIST: + case MDB_NOTFOUND: + break; // map to nothing generic + case MDB_PAGE_NOTFOUND: + case MDB_CORRUPTED: + return std::errc::state_not_recoverable; + case MDB_PANIC: + case MDB_VERSION_MISMATCH: + case MDB_INVALID: + break; // map to nothing generic + case MDB_MAP_FULL: + return std::errc::no_buffer_space; + case MDB_DBS_FULL: + break; // map to nothing generic + case MDB_READERS_FULL: + case MDB_TLS_FULL: + return std::errc::no_lock_available; + case MDB_TXN_FULL: + case MDB_CURSOR_FULL: + case MDB_PAGE_FULL: + case MDB_MAP_RESIZED: + break; // map to nothing generic + case MDB_INCOMPATIBLE: + return std::errc::invalid_argument; + case MDB_BAD_RSLOT: + case MDB_BAD_TXN: + case MDB_BAD_VALSIZE: + case MDB_BAD_DBI: + return std::errc::invalid_argument; + default: + return std::error_condition{value, std::generic_category()}; + } + return std::error_condition{value, *this}; + } + }; +} + +namespace lmdb +{ + std::error_category const& error_category() noexcept + { + static const category instance{}; + return instance; + } +} + diff --git a/src/lmdb/error.h b/src/lmdb/error.h new file mode 100644 index 000000000..2944adf78 --- /dev/null +++ b/src/lmdb/error.h @@ -0,0 +1,64 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <system_error> +#include <type_traits> + +//! Executes a LMDB command, and returns errors via `lmdb::error` enum. +#define MONERO_LMDB_CHECK(...) \ + do \ + { \ + const int err = __VA_ARGS__ ; \ + if (err) \ + return {lmdb::error(err)}; \ + } while (0) + +namespace lmdb +{ + //! Tracks LMDB error codes. + enum class error : int + { + // 0 is reserved for no error, as per expect<T> + // All other errors are the values reported by LMDB + }; + + std::error_category const& error_category() noexcept; + + inline std::error_code make_error_code(error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::lmdb::error> + : true_type + {}; +} diff --git a/src/lmdb/key_stream.h b/src/lmdb/key_stream.h new file mode 100644 index 000000000..40434d3a1 --- /dev/null +++ b/src/lmdb/key_stream.h @@ -0,0 +1,264 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "lmdb/value_stream.h" +#include "span.h" + +namespace lmdb +{ + + /*! + An InputIterator for a fixed-sized LMDB key and value. `operator++` + iterates over keys. + + \tparam K Key type in database records. + \tparam V Value type in database records. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename K, typename V> + class key_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> key; + + void increment() + { + // MDB_NEXT_MULTIPLE doesn't work if only one value is stored :/ + if (cur) + key = lmdb::stream::get(*cur, MDB_NEXT_NODUP, sizeof(K), sizeof(V)).first; + } + + public: + using value_type = std::pair<K, boost::iterator_range<value_iterator<V>>>; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + key_iterator() noexcept + : cur(nullptr), key() + {} + + /*! + \param cur Iterate over keys starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + key_iterator(MDB_cursor* cur) + : cur(cur), key() + { + if (cur) + key = lmdb::stream::get(*cur, MDB_GET_CURRENT, sizeof(K), sizeof(V)).first; + } + + //! \return True if `this` is one-past the last key. + bool is_end() const noexcept { return key.empty(); } + + //! \return True iff `rhs` is referencing `this` key. + bool equal(key_iterator const& rhs) const noexcept + { + return + (key.empty() && rhs.key.empty()) || + key.data() == rhs.key.data(); + } + + /*! + Moves iterator to next key or end. Invalidates all prior copies of + the iterator. + */ + key_iterator& operator++() + { + increment(); + return *this; + } + + /*! + Moves iterator to next key or end. + + \return A copy that is already invalidated, ignore + */ + key_iterator operator++(int) + { + key_iterator out{*this}; + increment(); + return out; + } + + //! \pre `!is_end()` \return {current key, current value range} + value_type operator*() const + { + return {get_key(), make_value_range()}; + } + + //! \pre `!is_end()` \return Current key + K get_key() const noexcept + { + assert(!is_end()); + K out; + std::memcpy(std::addressof(out), key.data(), sizeof(out)); + return out; + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return an entire value (`V`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + value_iterator<T, F, offset> make_value_iterator() const + { + static_assert(std::is_same<T, V>(), "bad MONERO_FIELD usage?"); + return {cur}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + boost::iterator_range<value_iterator<T, F, offset>> make_value_range() const + { + return {make_value_iterator<T, F, offset>(), value_iterator<T, F, offset>{}}; + } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized key `K` and + value `V`. + + \tparam K key type being stored by each record. + \tparam V value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename K, typename V, typename D> + class key_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit key_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + key_stream(key_stream&&) = default; + key_stream(key_stream const&) = delete; + ~key_stream() = default; + key_stream& operator=(key_stream&&) = default; + key_stream& operator=(key_stream const&) = delete; + + /*! + Give up ownership of the cursor. `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first key/value. Newly created + iterators will start at the first value again. + + \note Invalidates all current iterators, including those created + with `make_iterator` or `make_range`. Also invalidates all + `value_iterator`s created with `key_iterator`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator over database keys from current cursor + position that will reach `.is_end()` after the last key. + */ + key_iterator<K, V> make_iterator() const + { + return {cur.get()}; + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Range from current cursor position until last key record. + Useful in for-each range loops or in templated code + */ + boost::iterator_range<key_iterator<K, V>> make_range() const + { + return {make_iterator(), key_iterator<K, V>{}}; + } + }; + + template<typename K, typename V> + inline + bool operator==(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename K, typename V> + inline + bool operator!=(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/lmdb/table.cpp b/src/lmdb/table.cpp new file mode 100644 index 000000000..0818b74e6 --- /dev/null +++ b/src/lmdb/table.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "table.h" + +namespace lmdb +{ + expect<MDB_dbi> table::open(MDB_txn& write_txn) const noexcept + { + MONERO_PRECOND(name != nullptr); + + MDB_dbi out; + MONERO_LMDB_CHECK(mdb_dbi_open(&write_txn, name, flags, &out)); + if (key_cmp && !(flags & MDB_INTEGERKEY)) + MONERO_LMDB_CHECK(mdb_set_compare(&write_txn, out, key_cmp)); + if (value_cmp && !(flags & MDB_INTEGERDUP)) + MONERO_LMDB_CHECK(mdb_set_dupsort(&write_txn, out, value_cmp)); + return out; + } +} diff --git a/src/lmdb/table.h b/src/lmdb/table.h new file mode 100644 index 000000000..41a3de296 --- /dev/null +++ b/src/lmdb/table.h @@ -0,0 +1,120 @@ +#pragma once + +#include <utility> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/key_stream.h" +#include "lmdb/util.h" +#include "lmdb/value_stream.h" + +namespace lmdb +{ + //! Helper for grouping typical LMDB DBI options. + struct table + { + char const* const name; + const unsigned flags; + MDB_cmp_func const* const key_cmp; + MDB_cmp_func const* const value_cmp; + + //! \pre `name != nullptr` \return Open table. + expect<MDB_dbi> open(MDB_txn& write_txn) const noexcept; + }; + + //! Helper for grouping typical LMDB DBI options when key and value are fixed types. + template<typename K, typename V> + struct basic_table : table + { + using key_type = K; + using value_type = V; + + //! \return Additional LMDB flags based on `flags` value. + static constexpr unsigned compute_flags(const unsigned flags) noexcept + { + return flags | ((flags & MDB_DUPSORT) ? MDB_DUPFIXED : 0); + } + + constexpr explicit basic_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept + : table{name, compute_flags(flags), &lmdb::less<lmdb::native_type<K>>, value_cmp} + {} + + /*! + \tparam U must be same as `V`; used for sanity checking. + \tparam F is the type within `U` that is being extracted. + \tparam offset to `F` within `U`. + + \note If using `F` and `offset` to retrieve a specific field, use + `MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the + offset automatically. + + \return Value of type `F` at `offset` within `value` which has + type `U`. + */ + template<typename U, typename F = U, std::size_t offset = 0> + static expect<F> get_value(MDB_val value) noexcept + { + static_assert(std::is_same<U, V>(), "bad MONERO_FIELD?"); + static_assert(std::is_pod<F>(), "F must be POD"); + static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); + + if (value.mv_size != sizeof(U)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + F out; + std::memcpy(std::addressof(out), static_cast<char*>(value.mv_data) + offset, sizeof(out)); + return out; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first key/value in the table linked + to `cur` or an empty `key_stream`. + */ + template<typename D> + expect<key_stream<K, V, D>> + static get_key_stream(std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key; + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return key_stream<K, V, D>{std::move(cur)}; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first value at `key` in the table linked + to `cur` or an empty `value_stream`. + */ + template<typename D> + expect<value_stream<V, D>> + static get_value_stream(K const& key, std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key_bytes = lmdb::to_val(key); + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key_bytes, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return value_stream<V, D>{std::move(cur)}; + } + }; +} // lmdb + diff --git a/src/lmdb/transaction.h b/src/lmdb/transaction.h new file mode 100644 index 000000000..cdd80696c --- /dev/null +++ b/src/lmdb/transaction.h @@ -0,0 +1,95 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <lmdb.h> +#include <memory> + +#include "lmdb/error.h" + +//! Uses C++ type system to differentiate between cursors +#define MONERO_CURSOR(name) \ + struct close_ ## name : ::lmdb::close_cursor {}; \ + using name = std::unique_ptr< MDB_cursor, close_ ## name >; + +namespace lmdb +{ + struct abort_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + mdb_txn_abort(ptr); + } + }; + + /*! + Only valid if used via `create_read_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct release_read_txn + { + void operator()(MDB_txn* ptr) const noexcept; + // implementation in database.cpp + }; + + /*! + Only valid if used via `create_write_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct abort_write_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + release_read_txn{}(ptr); + } + }; + + struct close_cursor + { + void operator()(MDB_cursor* ptr) const noexcept + { + if (ptr) + mdb_cursor_close(ptr); + } + }; + + template<typename D> + inline expect<std::unique_ptr<MDB_cursor, D>> + open_cursor(MDB_txn& txn, MDB_dbi tbl) noexcept + { + MDB_cursor* cur = nullptr; + MONERO_LMDB_CHECK(mdb_cursor_open(&txn, tbl, &cur)); + return std::unique_ptr<MDB_cursor, D>{cur}; + } + + // The below use the C++ type system to designate `MDB_txn` status. + + using suspended_txn = std::unique_ptr<MDB_txn, abort_txn>; + using read_txn = std::unique_ptr<MDB_txn, release_read_txn>; + using write_txn = std::unique_ptr<MDB_txn, abort_write_txn>; +} // lmdb diff --git a/src/lmdb/util.h b/src/lmdb/util.h new file mode 100644 index 000000000..50162b7c8 --- /dev/null +++ b/src/lmdb/util.h @@ -0,0 +1,149 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <cstddef> +#include <cstring> +#include <lmdb.h> +#include <type_traits> +#include <utility> + +#include "span.h" + +/*! Calculates types and offset of struct field. Use in template arguments for + `table::get_value`, `value_iterator::get_value`, + `value_stream::make_iterator`, or `value_stream::make_range`. */ +#define MONERO_FIELD(obj, field) \ + obj , decltype(std::declval<obj>().field) , offsetof(obj, field) + +//! Expands to `lmdb::less` for the value `field` within `obj`. +#define MONERO_SORT_BY(obj, field) \ + &::lmdb::less< \ + lmdb::native_type<decltype(std::declval<obj>().field)>, \ + offsetof(obj, field) \ + > + +//! Expands to `lmdb::compare` for the value `field` within `obj`. +#define MONERO_COMPARE(obj, field) \ + &::lmdb::compare< \ + decltype(std::declval<obj>().field), \ + offsetof(obj, field) \ + > + +namespace lmdb +{ + //! Prevent instantiation of `std::underlying_type<T>` when `T` is not enum. + template<typename T> + struct identity + { + using type = T; + }; + + /*! + Get the native type for enums, or return `T` unchanged. Useful for + merging generated machine code for templated functions that use enums + with identical size-widths without relying on aggressive identical + comdat folding (ICF) support in linker. So with enum defintion + `enum class enum_foo : unsigned long {};` will always yield + `assert(&func_foo<unsigned long> == &func_foo<native_type<enum_foo>>)`. + */ + template<typename T> + using native_type = typename std::conditional< + std::is_enum<T>::value, std::underlying_type<T>, identity<T> + >::type::type; + + //! \return `value` as its native type. + template<typename T, typename U = typename std::underlying_type<T>::type> + inline constexpr U to_native(T value) noexcept + { + return U(value); + } + + //! \return `value` bytes in a LMDB `MDB_val` object. + template<typename T> + inline MDB_val to_val(T&& value) noexcept + { + // lmdb does not touch user data, so const_cast is acceptable + static_assert(!std::is_rvalue_reference<T&&>(), "cannot use temporary value"); + void const* const temp = reinterpret_cast<void const*>(std::addressof(value)); + return MDB_val{sizeof(value), const_cast<void*>(temp)}; + } + + //! \return A span over the same chunk of memory as `value`. + inline constexpr epee::span<const std::uint8_t> to_byte_span(MDB_val value) noexcept + { + return {static_cast<const std::uint8_t*>(value.mv_data), value.mv_size}; + } + + /*! + A LMDB comparison function that uses `operator<`. + + \tparam T has a defined `operator<` . + \tparam offset to `T` within the value. + + \return -1 if `left < right`, 1 if `right < left`, and 0 otherwise. + */ + template<typename T, std::size_t offset = 0> + inline int less(MDB_val const* left, MDB_val const* right) noexcept + { + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + + T left_val; + T right_val; + std::memcpy(std::addressof(left_val), static_cast<char*>(left->mv_data) + offset, sizeof(T)); + std::memcpy(std::addressof(right_val), static_cast<char*>(right->mv_data) + offset, sizeof(T)); + return left_val < right_val ? -1 : bool(right_val < left_val); + } + + /*! + A LMDB comparison function that uses `std::memcmp`. + + \toaram T is `!epee::has_padding` + \tparam offset to `T` within the value. + + \return The result of `std::memcmp` over the value. + */ + template<typename T, std::size_t offset = 0> + inline int compare(MDB_val const* left, MDB_val const* right) noexcept + { + static_assert(!epee::has_padding<T>(), "memcmp will not work"); + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + return std::memcmp( + static_cast<char*>(left->mv_data) + offset, + static_cast<char*>(right->mv_data) + offset, + sizeof(T) + ); + } +} // lmdb diff --git a/src/lmdb/value_stream.cpp b/src/lmdb/value_stream.cpp new file mode 100644 index 000000000..1024deb06 --- /dev/null +++ b/src/lmdb/value_stream.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "value_stream.h" + +#include <stdexcept> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +namespace lmdb +{ + namespace stream + { + std::size_t count(MDB_cursor* cur) + { + std::size_t out = 0; + if (cur) + { + const int rc = mdb_cursor_count(cur, &out); + if (rc) + MONERO_THROW(lmdb::error(rc), "mdb_cursor_count"); + } + return out; + } + + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value) + { + MDB_val key_bytes{}; + MDB_val value_bytes{}; + const int rc = mdb_cursor_get(&cur, &key_bytes, &value_bytes, op); + if (rc) + { + if (rc == MDB_NOTFOUND) + return {}; + MONERO_THROW(lmdb::error(rc), "mdb_cursor_get"); + } + + if (key && key != key_bytes.mv_size) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get key"); + + if (value && (value_bytes.mv_size % value != 0 || value_bytes.mv_size == 0)) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get value"); + + return {lmdb::to_byte_span(key_bytes), lmdb::to_byte_span(value_bytes)}; + } + } +} + diff --git a/src/lmdb/value_stream.h b/src/lmdb/value_stream.h new file mode 100644 index 000000000..c9977221f --- /dev/null +++ b/src/lmdb/value_stream.h @@ -0,0 +1,287 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "span.h" + +namespace lmdb +{ + namespace stream + { + /* + \throw std::system_error if unexpected LMDB error. + \return 0 if `cur == nullptr`, otherwise count of values at current key. + */ + std::size_t count(MDB_cursor* cur); + + /*! + Calls `mdb_cursor_get` and does some error checking. + + \param cur is given to `mdb_cursor_get` without modification. + \param op is passed to `mdb_cursor_get` without modification. + \param key expected key size or 0 to skip key size check. + \param value expected value size or 0 to skip value size check. + + \throw std::system_error if `key != 0` and `key_.mv_size != key`. + \throw std::system_error if `value != 0` and `value_.mv_size != value`. + \throw std::system_error if `mdb_cursor_get` returns any error + other than `MDB_NOTFOUND`. + + \return {key bytes, value bytes} or two empty spans if `MDB_NOTFOUND`. + */ + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value); + } + + /*! + An InputIterator for a fixed-sized LMDB value at a specific key. + + \tparam T The value type at the specific key. + \tparam F The value type being returned when dereferenced. + \tparam offset to `F` within `T`. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename T, typename F = T, std::size_t offset = 0> + class value_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> values; + + void increment() + { + values.remove_prefix(sizeof(T)); + if (values.empty() && cur) + values = lmdb::stream::get(*cur, MDB_NEXT_DUP, 0, sizeof(T)).second; + } + + public: + using value_type = F; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + value_iterator() noexcept + : cur(nullptr), values() + {} + + /*! + \param cur Iterate over values starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + value_iterator(MDB_cursor* cur) + : cur(cur), values() + { + if (cur) + values = lmdb::stream::get(*cur, MDB_GET_CURRENT, 0, sizeof(T)).second; + } + + value_iterator(value_iterator const&) = default; + ~value_iterator() = default; + value_iterator& operator=(value_iterator const&) = default; + + //! \return True if `this` is one-past the last value. + bool is_end() const noexcept { return values.empty(); } + + //! \return True iff `rhs` is referencing `this` value. + bool equal(value_iterator const& rhs) const noexcept + { + return + (values.empty() && rhs.values.empty()) || + values.data() == rhs.values.data(); + } + + //! Invalidates all prior copies of the iterator. + value_iterator& operator++() + { + increment(); + return *this; + } + + //! \return A copy that is already invalidated, ignore + value_iterator operator++(int) + { + value_iterator out{*this}; + increment(); + return out; + } + + /*! + Get a specific field within `F`. Default behavior is to return + the entirety of `U`, despite the filtering logic of `operator*`. + + \pre `!is_end()` + + \tparam U must match `T`, used for `MONERO_FIELD` sanity checking. + \tparam G field type to extract from the value + \tparam uoffset to `G` type, or `0` when `std::is_same<U, G>()`. + + \return The field `G`, at `uoffset` within `U`. + */ + template<typename U, typename G = U, std::size_t uoffset = 0> + G get_value() const noexcept + { + static_assert(std::is_same<U, T>(), "bad MONERO_FIELD usage?"); + static_assert(std::is_pod<U>(), "value type must be pod"); + static_assert(std::is_pod<G>(), "field type must be pod"); + static_assert(sizeof(G) + uoffset <= sizeof(U), "bad field and/or offset"); + assert(sizeof(G) + uoffset <= values.size()); + assert(!is_end()); + + G value; + std::memcpy(std::addressof(value), values.data() + uoffset, sizeof(value)); + return value; + } + + //! \pre `!is_end()` \return The field `F`, at `offset`, within `T`. + value_type operator*() const noexcept { return get_value<T, F, offset>(); } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized value `T`. + + \tparam T value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename T, typename D> + class value_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit value_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + value_stream(value_stream&&) = default; + value_stream(value_stream const&) = delete; + ~value_stream() = default; + value_stream& operator=(value_stream&&) = default; + value_stream& operator=(value_stream const&) = delete; + + /*! + Give up ownership of the cursor. `count()`, `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first value. Newly created iterators + will start at the first value again. + + \note Invalidates all current iterators from `this`, including + those created with `make_iterator` or `make_range`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST_DUP, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Number of values at this key. + */ + std::size_t count() const + { + return lmdb::stream::count(cur.get()); + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return entire value (`T`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + value_iterator<U, F, offset> make_iterator() const + { + static_assert(std::is_same<U, T>(), "was MONERO_FIELD used with wrong type?"); + return {cur.get()}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + boost::iterator_range<value_iterator<U, F, offset>> make_range() const + { + return {make_iterator<U, F, offset>(), value_iterator<U, F, offset>{}}; + } + }; + + template<typename T, typename F, std::size_t offset> + inline + bool operator==(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename T, typename F, std::size_t offset> + inline + bool operator!=(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/mnemonics/CMakeLists.txt b/src/mnemonics/CMakeLists.txt index e3836bcca..acb9a4338 100644 --- a/src/mnemonics/CMakeLists.txt +++ b/src/mnemonics/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/mnemonics/chinese_simplified.h b/src/mnemonics/chinese_simplified.h index 0566b1079..ff035dde0 100644 --- a/src/mnemonics/chinese_simplified.h +++ b/src/mnemonics/chinese_simplified.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
-// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project
+// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/dutch.h b/src/mnemonics/dutch.h index 801caf986..14ba56a7e 100644 --- a/src/mnemonics/dutch.h +++ b/src/mnemonics/dutch.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index b1e3bdcd5..48c9ab1ba 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -76,8 +76,8 @@ namespace crypto namespace { uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, - uint32_t unique_prefix_length); - bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length); + const Language::Base *language); + bool checksum_test(std::vector<epee::wipeable_string> seed, const Language::Base *language); /*! * \brief Finds the word list that contains the seed words and puts the indices @@ -116,8 +116,8 @@ namespace for (std::vector<Language::Base*>::iterator it1 = language_instances.begin(); it1 != language_instances.end(); it1++) { - const std::unordered_map<epee::wipeable_string, uint32_t> &word_map = (*it1)->get_word_map(); - const std::unordered_map<epee::wipeable_string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t, Language::WordHash, Language::WordEqual> &word_map = (*it1)->get_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t, Language::WordHash, Language::WordEqual> &trimmed_word_map = (*it1)->get_trimmed_word_map(); // To iterate through seed words bool full_match = true; @@ -151,7 +151,7 @@ namespace // if we were using prefix only, and we have a checksum, check it now // to avoid false positives due to prefix set being too common if (has_checksum) - if (!checksum_test(seed, (*it1)->get_unique_prefix_length())) + if (!checksum_test(seed, *it1)) { fallback = *it1; full_match = false; @@ -190,20 +190,20 @@ namespace * \return Checksum index */ uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, - uint32_t unique_prefix_length) + const Language::Base *language) { - epee::wipeable_string trimmed_words = ""; + epee::wipeable_string trimmed_words = "", word; + const auto &word_map = language->get_word_map(); + const auto &trimmed_word_map = language->get_trimmed_word_map(); + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) { - if (it->length() > unique_prefix_length) - { - trimmed_words += Language::utf8prefix(*it, unique_prefix_length); - } - else - { - trimmed_words += *it; - } + word = Language::utf8prefix(*it, unique_prefix_length); + auto it2 = trimmed_word_map.find(word); + if (it2 == trimmed_word_map.end()) + throw std::runtime_error("Word \"" + std::string(word.data(), word.size()) + "\" not found in trimmed word map in " + language->get_english_language_name()); + trimmed_words += it2->first; } boost::crc_32_type result; result.process_bytes(trimmed_words.data(), trimmed_words.length()); @@ -216,7 +216,7 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return True if the test passed false if not. */ - bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length) + bool checksum_test(std::vector<epee::wipeable_string> seed, const Language::Base *language) { if (seed.empty()) return false; @@ -224,13 +224,16 @@ namespace epee::wipeable_string last_word = seed.back(); seed.pop_back(); - epee::wipeable_string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); + + auto idx = create_checksum_index(seed, language); + epee::wipeable_string checksum = seed[idx]; epee::wipeable_string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : checksum; epee::wipeable_string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; - bool ret = trimmed_checksum == trimmed_last_word; + bool ret = Language::WordEqual()(trimmed_checksum, trimmed_last_word); MINFO("Checksum is " << (ret ? "valid" : "invalid")); return ret; } @@ -301,7 +304,7 @@ namespace crypto if (has_checksum) { - if (!checksum_test(seed, language->get_unique_prefix_length())) + if (!checksum_test(seed, language)) { // Checksum fail MERROR("Invalid seed: invalid checksum"); @@ -424,7 +427,7 @@ namespace crypto memwipe(w, sizeof(w)); } - words += words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]; + words += words_store[create_checksum_index(words_store, language)]; return true; } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 60d2c5f15..9aa727e45 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/english.h b/src/mnemonics/english.h index d5c5594ef..677b70052 100644 --- a/src/mnemonics/english.h +++ b/src/mnemonics/english.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/english_old.h b/src/mnemonics/english_old.h index e35b907df..179fc45ec 100644 --- a/src/mnemonics/english_old.h +++ b/src/mnemonics/english_old.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin
//
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/esperanto.h b/src/mnemonics/esperanto.h index b0be235ed..ef63a2235 100644 --- a/src/mnemonics/esperanto.h +++ b/src/mnemonics/esperanto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/french.h b/src/mnemonics/french.h index 48ec46f78..85925aa53 100644 --- a/src/mnemonics/french.h +++ b/src/mnemonics/french.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/german.h b/src/mnemonics/german.h index 883a173a3..40116970c 100644 --- a/src/mnemonics/german.h +++ b/src/mnemonics/german.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor Shrikez
//
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/italian.h b/src/mnemonics/italian.h index 57cdfa25e..204fdcb78 100644 --- a/src/mnemonics/italian.h +++ b/src/mnemonics/italian.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor Shrikez
//
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/japanese.h b/src/mnemonics/japanese.h index 5baabedf2..6213c4976 100644 --- a/src/mnemonics/japanese.h +++ b/src/mnemonics/japanese.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
-// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project
+// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 52e784cef..653314b04 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
@@ -38,7 +38,9 @@ #include <vector>
#include <unordered_map>
#include <string>
+#include <boost/algorithm/string.hpp>
#include "misc_log_ex.h"
+#include "fnv1.h"
/*!
* \namespace Language
@@ -71,6 +73,92 @@ namespace Language return prefix;
}
+ template<typename T>
+ inline T utf8canonical(const T &s)
+ {
+ T sc = "";
+ size_t avail = s.size();
+ const char *ptr = s.data();
+ wint_t cp = 0;
+ int bytes = 1;
+ char wbuf[8], *wptr;
+ while (avail--)
+ {
+ if ((*ptr & 0x80) == 0)
+ {
+ cp = *ptr++;
+ bytes = 1;
+ }
+ else if ((*ptr & 0xe0) == 0xc0)
+ {
+ if (avail < 1)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0x1f) << 6;
+ cp |= *ptr++ & 0x3f;
+ --avail;
+ bytes = 2;
+ }
+ else if ((*ptr & 0xf0) == 0xe0)
+ {
+ if (avail < 2)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0xf) << 12;
+ cp |= (*ptr++ & 0x3f) << 6;
+ cp |= *ptr++ & 0x3f;
+ avail -= 2;
+ bytes = 3;
+ }
+ else if ((*ptr & 0xf8) == 0xf0)
+ {
+ if (avail < 3)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0x7) << 18;
+ cp |= (*ptr++ & 0x3f) << 12;
+ cp |= (*ptr++ & 0x3f) << 6;
+ cp |= *ptr++ & 0x3f;
+ avail -= 3;
+ bytes = 4;
+ }
+ else
+ throw std::runtime_error("Invalid UTF-8");
+
+ cp = std::towlower(cp);
+ wptr = wbuf;
+ switch (bytes)
+ {
+ case 1: *wptr++ = cp; break;
+ case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr += 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ default: throw std::runtime_error("Invalid UTF-8");
+ }
+ *wptr = 0;
+ sc += T(wbuf, bytes);
+ cp = 0;
+ bytes = 1;
+ }
+ return sc;
+ }
+
+ struct WordHash
+ {
+ std::size_t operator()(const epee::wipeable_string &s) const
+ {
+ const epee::wipeable_string sc = utf8canonical(s);
+ return epee::fnv::FNV1a(sc.data(), sc.size());
+ }
+ };
+
+ struct WordEqual
+ {
+ bool operator()(const epee::wipeable_string &s0, const epee::wipeable_string &s1) const
+ {
+ const epee::wipeable_string s0c = utf8canonical(s0);
+ const epee::wipeable_string s1c = utf8canonical(s1);
+ return s0c == s1c;
+ }
+ };
+
/*!
* \class Base
* \brief A base language class which all languages have to inherit from for
@@ -87,8 +175,8 @@ namespace Language NWORDS = 1626
};
std::vector<std::string> word_list; /*!< A pointer to the array of words */
- std::unordered_map<epee::wipeable_string, uint32_t> word_map; /*!< hash table to find word's index */
- std::unordered_map<epee::wipeable_string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
+ std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual> word_map; /*!< hash table to find word's index */
+ std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual> trimmed_word_map; /*!< hash table to find word's trimmed index */
std::string language_name; /*!< Name of language */
std::string english_language_name; /*!< Name of language */
uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
@@ -159,7 +247,7 @@ namespace Language * \brief Returns a pointer to the word map.
* \return A pointer to the word map.
*/
- const std::unordered_map<epee::wipeable_string, uint32_t>& get_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual>& get_word_map() const
{
return word_map;
}
@@ -167,7 +255,7 @@ namespace Language * \brief Returns a pointer to the trimmed word map.
* \return A pointer to the trimmed word map.
*/
- const std::unordered_map<epee::wipeable_string, uint32_t>& get_trimmed_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual>& get_trimmed_word_map() const
{
return trimmed_word_map;
}
diff --git a/src/mnemonics/lojban.h b/src/mnemonics/lojban.h index 5162a8ec9..b6baf46c4 100644 --- a/src/mnemonics/lojban.h +++ b/src/mnemonics/lojban.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/portuguese.h b/src/mnemonics/portuguese.h index af04f89c2..dbc51fb2a 100644 --- a/src/mnemonics/portuguese.h +++ b/src/mnemonics/portuguese.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
-// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project
+// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/russian.h b/src/mnemonics/russian.h index f3e70ede6..d36942a9b 100644 --- a/src/mnemonics/russian.h +++ b/src/mnemonics/russian.h @@ -1,6 +1,6 @@ // Word list created by Monero contributor sammy007
//
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/singleton.h b/src/mnemonics/singleton.h index d317a2d8d..ffe3fe4de 100644 --- a/src/mnemonics/singleton.h +++ b/src/mnemonics/singleton.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/spanish.h b/src/mnemonics/spanish.h index 4d7a896a6..a05485775 100644 --- a/src/mnemonics/spanish.h +++ b/src/mnemonics/spanish.h @@ -21,7 +21,7 @@ // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
-// Code surrounding the word list is Copyright (c) 2014-2018, The Monero Project
+// Code surrounding the word list is Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
diff --git a/src/multisig/CMakeLists.txt b/src/multisig/CMakeLists.txt index a770c6dc5..4631b75ef 100644 --- a/src/multisig/CMakeLists.txt +++ b/src/multisig/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2018, The Monero Project +# Copyright (c) 2017-2019, The Monero Project # # All rights reserved. # diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index 33d0a312f..14df4d554 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/multisig/multisig.h b/src/multisig/multisig.h index 93a756812..bc6edbb05 100644 --- a/src/multisig/multisig.h +++ b/src/multisig/multisig.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt new file mode 100644 index 000000000..8a3ee9e6f --- /dev/null +++ b/src/net/CMakeLists.txt @@ -0,0 +1,34 @@ +# Copyright (c) 2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(net_sources error.cpp parse.cpp socks.cpp tor_address.cpp i2p_address.cpp) +set(net_headers error.h parse.h socks.h tor_address.h i2p_address.h) + +monero_add_library(net ${net_sources} ${net_headers}) +target_link_libraries(net common epee ${Boost_ASIO_LIBRARY}) + diff --git a/src/net/error.cpp b/src/net/error.cpp new file mode 100644 index 000000000..037f44d52 --- /dev/null +++ b/src/net/error.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +#include <string> + +namespace +{ + struct net_category : std::error_category + { + net_category() noexcept + : std::error_category() + {} + + const char* name() const noexcept override + { + return "net::error_category"; + } + + std::string message(int value) const override + { + switch (net::error(value)) + { + case net::error::expected_tld: + return "Expected top-level domain"; + case net::error::invalid_host: + return "Host value is not valid"; + case net::error::invalid_i2p_address: + return "Invalid I2P address"; + case net::error::invalid_port: + return "Invalid port value (expected 0-65535)"; + case net::error::invalid_tor_address: + return "Invalid Tor address"; + case net::error::unsupported_address: + return "Network address not supported"; + default: + break; + } + + return "Unknown net::error"; + } + + std::error_condition default_error_condition(int value) const noexcept override + { + switch (net::error(value)) + { + case net::error::invalid_port: + return std::errc::result_out_of_range; + case net::error::expected_tld: + case net::error::invalid_tor_address: + default: + break; + } + return std::error_condition{value, *this}; + } + }; +} // anonymous + +namespace net +{ + std::error_category const& error_category() noexcept + { + static const net_category instance{}; + return instance; + } +} diff --git a/src/net/error.h b/src/net/error.h new file mode 100644 index 000000000..c8338f7e2 --- /dev/null +++ b/src/net/error.h @@ -0,0 +1,64 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <system_error> +#include <type_traits> + +namespace net +{ + //! General net errors + enum class error : int + { + // 0 reserved for success (as per expect<T>) + expected_tld = 1, //!< Expected a tld + invalid_host, //!< Hostname is not valid + invalid_i2p_address, + invalid_port, //!< Outside of 0-65535 range + invalid_tor_address,//!< Invalid base32 or length + unsupported_address //!< Type not supported by `get_network_address` + }; + + //! \return `std::error_category` for `net` namespace. + std::error_category const& error_category() noexcept; + + //! \return `net::error` as a `std::error_code` value. + inline std::error_code make_error_code(error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::net::error> + : true_type + {}; +} diff --git a/src/net/fwd.h b/src/net/fwd.h new file mode 100644 index 000000000..7cae88251 --- /dev/null +++ b/src/net/fwd.h @@ -0,0 +1,46 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cstdint> + +namespace net +{ + enum class error : int; + class tor_address; + class i2p_address; + + namespace socks + { + class client; + template<typename> class connect_handler; + enum class error : int; + enum class version : std::uint8_t; + } +} diff --git a/src/net/i2p_address.cpp b/src/net/i2p_address.cpp new file mode 100644 index 000000000..cba829d3f --- /dev/null +++ b/src/net/i2p_address.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "i2p_address.h" + +#include <algorithm> +#include <boost/spirit/include/karma_generate.hpp> +#include <boost/spirit/include/karma_uint.hpp> +#include <cassert> +#include <cstring> +#include <limits> + +#include "net/error.h" +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage.h" +#include "string_tools.h" + +namespace net +{ + namespace + { + // !TODO only b32 addresses right now + constexpr const char tld[] = u8".b32.i2p"; + constexpr const char unknown_host[] = "<unknown i2p host>"; + + constexpr const unsigned b32_length = 52; + + constexpr const char base32_alphabet[] = + u8"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz234567"; + + expect<void> host_check(boost::string_ref host) noexcept + { + if (!host.ends_with(tld)) + return {net::error::expected_tld}; + + host.remove_suffix(sizeof(tld) - 1); + + if (host.size() != b32_length) + return {net::error::invalid_i2p_address}; + if (host.find_first_not_of(base32_alphabet) != boost::string_ref::npos) + return {net::error::invalid_i2p_address}; + + return success(); + } + + struct i2p_serialized + { + std::string host; + std::uint16_t port; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(host) + KV_SERIALIZE(port) + END_KV_SERIALIZE_MAP() + }; + } + + i2p_address::i2p_address(const boost::string_ref host, const std::uint16_t port) noexcept + : port_(port) + { + // this is a private constructor, throw if moved to public + assert(host.size() < sizeof(host_)); + + const std::size_t length = std::min(sizeof(host_) - 1, host.size()); + std::memcpy(host_, host.data(), length); + std::memset(host_ + length, 0, sizeof(host_) - length); + } + + const char* i2p_address::unknown_str() noexcept + { + return unknown_host; + } + + i2p_address::i2p_address() noexcept + : port_(0) + { + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); + std::memset(host_ + sizeof(unknown_host), 0, sizeof(host_) - sizeof(unknown_host)); + } + + expect<i2p_address> i2p_address::make(const boost::string_ref address, const std::uint16_t default_port) + { + boost::string_ref host = address.substr(0, address.rfind(':')); + const boost::string_ref port = + address.substr(host.size() + (host.size() == address.size() ? 0 : 1)); + + MONERO_CHECK(host_check(host)); + + std::uint16_t porti = default_port; + if (!port.empty() && !epee::string_tools::get_xtype_from_string(porti, std::string{port})) + return {net::error::invalid_port}; + + static_assert(b32_length + sizeof(tld) == sizeof(i2p_address::host_), "bad internal host size"); + return i2p_address{host, porti}; + } + + bool i2p_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent) + { + i2p_serialized in{}; + if (in._load(src, hparent) && in.host.size() < sizeof(host_) && (in.host == unknown_host || !host_check(in.host).has_error())) + { + std::memcpy(host_, in.host.data(), in.host.size()); + std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); + port_ = in.port; + return true; + } + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator + port_ = 0; + return false; + } + + bool i2p_address::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const + { + const i2p_serialized out{std::string{host_}, port_}; + return out.store(dest, hparent); + } + + i2p_address::i2p_address(const i2p_address& rhs) noexcept + : port_(rhs.port_) + { + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + + i2p_address& i2p_address::operator=(const i2p_address& rhs) noexcept + { + if (this != std::addressof(rhs)) + { + port_ = rhs.port_; + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + return *this; + } + + bool i2p_address::is_unknown() const noexcept + { + static_assert(1 <= sizeof(host_), "host size too small"); + return host_[0] == '<'; // character is not allowed otherwise + } + + bool i2p_address::equal(const i2p_address& rhs) const noexcept + { + return port_ == rhs.port_ && is_same_host(rhs); + } + + bool i2p_address::less(const i2p_address& rhs) const noexcept + { + return std::strcmp(host_str(), rhs.host_str()) < 0 || port() < rhs.port(); + } + + bool i2p_address::is_same_host(const i2p_address& rhs) const noexcept + { + return std::strcmp(host_str(), rhs.host_str()) == 0; + } + + std::string i2p_address::str() const + { + const std::size_t host_length = std::strlen(host_str()); + const std::size_t port_length = + port_ == 0 ? 0 : std::numeric_limits<std::uint16_t>::digits10 + 2; + + std::string out{}; + out.reserve(host_length + port_length); + out.assign(host_str(), host_length); + + if (port_ != 0) + { + out.push_back(':'); + namespace karma = boost::spirit::karma; + karma::generate(std::back_inserter(out), karma::ushort_, port()); + } + return out; + } +} diff --git a/src/net/i2p_address.h b/src/net/i2p_address.h new file mode 100644 index 000000000..28a1118ba --- /dev/null +++ b/src/net/i2p_address.h @@ -0,0 +1,140 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/utility/string_ref.hpp> +#include <cstdint> +#include <string> + +#include "common/expect.h" +#include "net/enums.h" +#include "net/error.h" + +namespace epee +{ +namespace serialization +{ + class portable_storage; + struct section; +} +} + +namespace net +{ + //! b32 i2p address; internal format not condensed/decoded. + class i2p_address + { + std::uint16_t port_; + char host_[61]; // null-terminated + + //! Keep in private, `host.size()` has no runtime check + i2p_address(boost::string_ref host, std::uint16_t port) noexcept; + + public: + //! \return Size of internal buffer for host. + static constexpr std::size_t buffer_size() noexcept { return sizeof(host_); } + + //! \return `<unknown tor host>`. + static const char* unknown_str() noexcept; + + //! An object with `port() == 0` and `host_str() == unknown_str()`. + i2p_address() noexcept; + + //! \return A default constructed `i2p_address` object. + static i2p_address unknown() noexcept { return i2p_address{}; } + + /*! + Parse `address` in b32 i2p format (i.e. x.b32.i2p:80) + with `default_port` being used if port is not specified in + `address`. + */ + static expect<i2p_address> make(boost::string_ref address, std::uint16_t default_port = 0); + + //! Load from epee p2p format, and \return false if not valid tor address + bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent); + + //! Store in epee p2p format + bool store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const; + + // Moves and copies are currently identical + + i2p_address(const i2p_address& rhs) noexcept; + ~i2p_address() = default; + i2p_address& operator=(const i2p_address& rhs) noexcept; + + //! \return True if default constructed or via `unknown()`. + bool is_unknown() const noexcept; + + bool equal(const i2p_address& rhs) const noexcept; + bool less(const i2p_address& rhs) const noexcept; + + //! \return True if i2p addresses are identical. + bool is_same_host(const i2p_address& rhs) const noexcept; + + //! \return `x.b32.i2p` or `x.b32.i2p:z` if `port() != 0`. + std::string str() const; + + //! \return Null-terminated `x.b32.i2p` value or `unknown_str()`. + const char* host_str() const noexcept { return host_; } + + //! \return Port value or `0` if unspecified. + std::uint16_t port() const noexcept { return port_; } + + static constexpr bool is_loopback() noexcept { return false; } + static constexpr bool is_local() noexcept { return false; } + + static constexpr epee::net_utils::address_type get_type_id() noexcept + { + return epee::net_utils::address_type::i2p; + } + + static constexpr epee::net_utils::zone get_zone() noexcept + { + return epee::net_utils::zone::i2p; + } + + //! \return `!is_unknown()`. + bool is_blockable() const noexcept { return !is_unknown(); } + }; + + inline bool operator==(const i2p_address& lhs, const i2p_address& rhs) noexcept + { + return lhs.equal(rhs); + } + + inline bool operator!=(const i2p_address& lhs, const i2p_address& rhs) noexcept + { + return !lhs.equal(rhs); + } + + inline bool operator<(const i2p_address& lhs, const i2p_address& rhs) noexcept + { + return lhs.less(rhs); + } +} // net diff --git a/src/net/parse.cpp b/src/net/parse.cpp new file mode 100644 index 000000000..eaaadb67e --- /dev/null +++ b/src/net/parse.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "parse.h" + +#include "net/tor_address.h" +#include "net/i2p_address.h" +#include "string_tools.h" + +namespace net +{ + expect<epee::net_utils::network_address> + get_network_address(const boost::string_ref address, const std::uint16_t default_port) + { + const boost::string_ref host = address.substr(0, address.rfind(':')); + + if (host.empty()) + return make_error_code(net::error::invalid_host); + if (host.ends_with(".onion")) + return tor_address::make(address, default_port); + if (host.ends_with(".i2p")) + return i2p_address::make(address, default_port); + + std::uint16_t port = default_port; + if (host.size() < address.size()) + { + if (!epee::string_tools::get_xtype_from_string(port, std::string{address.substr(host.size() + 1)})) + return make_error_code(net::error::invalid_port); + } + + std::uint32_t ip = 0; + if (epee::string_tools::get_ip_int32_from_string(ip, std::string{host})) + return {epee::net_utils::ipv4_network_address{ip, port}}; + return make_error_code(net::error::unsupported_address); + } +} diff --git a/src/net/parse.h b/src/net/parse.h new file mode 100644 index 000000000..5804c4128 --- /dev/null +++ b/src/net/parse.h @@ -0,0 +1,54 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/utility/string_ref.hpp> +#include <cstdint> + +#include "common/expect.h" +#include "net/net_utils_base.h" + +namespace net +{ + /*! + Identifies onion, i2p and IPv4 addresses and returns them as a generic + `network_address`. If the type is unsupported, it might be a hostname, + and `error() == net::error::kUnsupportedAddress` is returned. + + \param address An onion address, i2p address, ipv4 address or hostname. Hostname + will return an error. + \param default_port If `address` does not specify a port, this value + will be used. + + \return A tor or IPv4 address, else error. + */ + expect<epee::net_utils::network_address> + get_network_address(boost::string_ref address, std::uint16_t default_port); +} + diff --git a/src/net/socks.cpp b/src/net/socks.cpp new file mode 100644 index 000000000..53154369b --- /dev/null +++ b/src/net/socks.cpp @@ -0,0 +1,316 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "socks.h" + +#include <algorithm> +#include <boost/asio/buffer.hpp> +#include <boost/asio/read.hpp> +#include <boost/asio/write.hpp> +#include <boost/endian/arithmetic.hpp> +#include <boost/endian/conversion.hpp> +#include <cstring> +#include <limits> +#include <string> + +#include "net/net_utils_base.h" +#include "net/tor_address.h" +#include "net/i2p_address.h" + +namespace net +{ +namespace socks +{ + namespace + { + constexpr const unsigned v4_reply_size = 8; + constexpr const std::uint8_t v4_connect_command = 1; + constexpr const std::uint8_t v4tor_resolve_command = 0xf0; + constexpr const std::uint8_t v4_request_granted = 90; + + struct v4_header + { + std::uint8_t version; + std::uint8_t command_code; + boost::endian::big_uint16_t port; + boost::endian::big_uint32_t ip; + }; + + std::size_t write_domain_header(epee::span<std::uint8_t> out, const std::uint8_t command, const std::uint16_t port, const boost::string_ref domain) + { + if (std::numeric_limits<std::size_t>::max() - sizeof(v4_header) - 2 < domain.size()) + return 0; + + const std::size_t buf_size = sizeof(v4_header) + domain.size() + 2; + if (out.size() < buf_size) + return 0; + + // version 4, 1 indicates invalid ip for domain extension + const v4_header temp{4, command, port, std::uint32_t(1)}; + std::memcpy(out.data(), std::addressof(temp), sizeof(temp)); + out.remove_prefix(sizeof(temp)); + + *(out.data()) = 0; + out.remove_prefix(1); + + std::memcpy(out.data(), domain.data(), domain.size()); + out.remove_prefix(domain.size()); + + *(out.data()) = 0; + return buf_size; + } + + struct socks_category : boost::system::error_category + { + explicit socks_category() noexcept + : boost::system::error_category() + {} + + const char* name() const noexcept override + { + return "net::socks::error_category"; + } + + virtual std::string message(int value) const override + { + switch (socks::error(value)) + { + case socks::error::rejected: + return "Socks request rejected or failed"; + case socks::error::identd_connection: + return "Socks request rejected because server cannot connect to identd on the client"; + case socks::error::identd_user: + return "Socks request rejected because the client program and identd report different user-ids"; + + case socks::error::bad_read: + return "Socks boost::async_read read fewer bytes than expected"; + case socks::error::bad_write: + return "Socks boost::async_write wrote fewer bytes than expected"; + case socks::error::unexpected_version: + return "Socks server returned unexpected version in reply"; + + default: + break; + } + return "Unknown net::socks::error"; + } + + boost::system::error_condition default_error_condition(int value) const noexcept override + { + switch (socks::error(value)) + { + case socks::error::bad_read: + case socks::error::bad_write: + return boost::system::errc::io_error; + case socks::error::unexpected_version: + return boost::system::errc::protocol_error; + default: + break; + }; + if (1 <= value && value <= 256) + return boost::system::errc::protocol_error; + + return boost::system::error_condition{value, *this}; + } + }; + } + + const boost::system::error_category& error_category() noexcept + { + static const socks_category instance{}; + return instance; + } + + struct client::completed + { + std::shared_ptr<client> self_; + + void operator()(const boost::system::error_code error, const std::size_t bytes) const + { + static_assert(1 < sizeof(self_->buffer_), "buffer too small for v4 response"); + + if (self_) + { + client& self = *self_; + self.buffer_size_ = std::min(bytes, sizeof(self.buffer_)); + + if (error) + self.done(error, std::move(self_)); + else if (self.buffer().size() < sizeof(v4_header)) + self.done(socks::error::bad_read, std::move(self_)); + else if (self.buffer_[0] != 0) // response version + self.done(socks::error::unexpected_version, std::move(self_)); + else if (self.buffer_[1] != v4_request_granted) + self.done(socks::error(int(self.buffer_[1]) + 1), std::move(self_)); + else + self.done(boost::system::error_code{}, std::move(self_)); + } + } + }; + + struct client::read + { + std::shared_ptr<client> self_; + + static boost::asio::mutable_buffers_1 get_buffer(client& self) noexcept + { + static_assert(sizeof(v4_header) <= sizeof(self.buffer_), "buffer too small for v4 response"); + return boost::asio::buffer(self.buffer_, sizeof(v4_header)); + } + + void operator()(const boost::system::error_code error, const std::size_t bytes) + { + if (self_) + { + client& self = *self_; + if (error) + self.done(error, std::move(self_)); + else if (bytes < self.buffer().size()) + self.done(socks::error::bad_write, std::move(self_)); + else + boost::asio::async_read(self.proxy_, get_buffer(self), completed{std::move(self_)}); + } + } + }; + + struct client::write + { + std::shared_ptr<client> self_; + + static boost::asio::const_buffers_1 get_buffer(client const& self) noexcept + { + return boost::asio::buffer(self.buffer_, self.buffer_size_); + } + + void operator()(const boost::system::error_code error) + { + if (self_) + { + client& self = *self_; + if (error) + self.done(error, std::move(self_)); + else + boost::asio::async_write(self.proxy_, get_buffer(self), read{std::move(self_)}); + } + } + }; + + client::client(stream_type::socket&& proxy, socks::version ver) + : proxy_(std::move(proxy)), buffer_size_(0), buffer_(), ver_(ver) + {} + + client::~client() {} + + bool client::set_connect_command(const epee::net_utils::ipv4_network_address& address) + { + switch (socks_version()) + { + case version::v4: + case version::v4a: + case version::v4a_tor: + break; + default: + return false; + } + + static_assert(sizeof(v4_header) < sizeof(buffer_), "buffer size too small for request"); + static_assert(0 < sizeof(buffer_), "buffer size too small for null termination"); + + // version 4 + const v4_header temp{4, v4_connect_command, address.port(), boost::endian::big_to_native(address.ip())}; + std::memcpy(std::addressof(buffer_), std::addressof(temp), sizeof(temp)); + buffer_[sizeof(temp)] = 0; + buffer_size_ = sizeof(temp) + 1; + + return true; + } + + bool client::set_connect_command(const boost::string_ref domain, std::uint16_t port) + { + switch (socks_version()) + { + case version::v4a: + case version::v4a_tor: + break; + + default: + return false; + } + + const std::size_t buf_used = write_domain_header(buffer_, v4_connect_command, port, domain); + buffer_size_ = buf_used; + return buf_used != 0; + } + + bool client::set_connect_command(const net::tor_address& address) + { + if (!address.is_unknown()) + return set_connect_command(address.host_str(), address.port()); + return false; + } + + bool client::set_connect_command(const net::i2p_address& address) + { + if (!address.is_unknown()) + return set_connect_command(address.host_str(), address.port()); + return false; + } + + bool client::set_resolve_command(boost::string_ref domain) + { + if (socks_version() != version::v4a_tor) + return false; + + const std::size_t buf_used = write_domain_header(buffer_, v4tor_resolve_command, 0, domain); + buffer_size_ = buf_used; + return buf_used != 0; + } + + bool client::connect_and_send(std::shared_ptr<client> self, const stream_type::endpoint& proxy_address) + { + if (self && !self->buffer().empty()) + { + client& alias = *self; + alias.proxy_.async_connect(proxy_address, write{std::move(self)}); + return true; + } + return false; + } + + bool client::send(std::shared_ptr<client> self) + { + if (self && !self->buffer().empty()) + { + client& alias = *self; + boost::asio::async_write(alias.proxy_, write::get_buffer(alias), read{std::move(self)}); + return true; + } + return false; + } +} // socks +} // net diff --git a/src/net/socks.h b/src/net/socks.h new file mode 100644 index 000000000..825937792 --- /dev/null +++ b/src/net/socks.h @@ -0,0 +1,228 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cstdint> +#include <boost/asio/ip/tcp.hpp> +#include <boost/asio/io_service.hpp> +#include <boost/system/error_code.hpp> +#include <boost/type_traits/integral_constant.hpp> +#include <boost/utility/string_ref.hpp> +#include <memory> +#include <utility> + +#include "net/fwd.h" +#include "span.h" + +namespace epee +{ +namespace net_utils +{ + class ipv4_network_address; +} +} + +namespace net +{ +namespace socks +{ + //! Supported socks variants. + enum class version : std::uint8_t + { + v4 = 0, + v4a, + v4a_tor //!< Extensions defined in Tor codebase + }; + + //! Possible errors with socks communication. Defined in https://www.openssh.com/txt/socks4.protocol + enum class error : int + { + // 0 is reserved for success value + // 1-256 -> reserved for error values from socks server (+1 from wire value). + rejected = 92, + identd_connection, + identd_user, + // Specific to application + bad_read = 257, + bad_write, + unexpected_version + }; + + /* boost::system::error_code is extended for easier compatibility with + boost::asio errors. If std::error_code is needed (with expect<T> for + instance), then upgrade to boost 1.65+ or use conversion code in + develop branch at boost/system/detail/std_interoperability.hpp */ + + //! \return boost::system::error_category for net::socks namespace + const boost::system::error_category& error_category() noexcept; + + //! \return net::socks::error as a boost::system::error_code. + inline boost::system::error_code make_error_code(error value) noexcept + { + return boost::system::error_code{int(value), socks::error_category()}; + } + + //! Client support for socks connect and resolve commands. + class client + { + boost::asio::ip::tcp::socket proxy_; + std::uint16_t buffer_size_; + std::uint8_t buffer_[1024]; + socks::version ver_; + + /*! + Only invoked after `*send(...)` function completes or fails. + `bool(error) == false` indicates success; `self.get()` is always + `this` and allows implementations to skip + `std::enable_shared_from_this<T>` (ASIO callbacks need shared_ptr). + The design saves space and reduces cycles (everything uses moves, + so no atomic operations are ever necessary). + + \param error when processing last command (if any). + \param self `shared_ptr<client>` handle to `this`. + */ + virtual void done(boost::system::error_code error, std::shared_ptr<client> self) = 0; + + public: + using stream_type = boost::asio::ip::tcp; + + // defined in cpp + struct write; + struct read; + struct completed; + + /*! + \param proxy ownership is passed into `this`. Does not have to be + in connected state. + \param ver socks version for the connection. + */ + explicit client(stream_type::socket&& proxy, socks::version ver); + + client(const client&) = delete; + virtual ~client(); + client& operator=(const client&) = delete; + + //! \return Ownership of socks client socket object. + stream_type::socket take_socket() + { + return stream_type::socket{std::move(proxy_)}; + } + + //! \return Socks version. + socks::version socks_version() const noexcept { return ver_; } + + //! \return Contents of internal buffer. + epee::span<const std::uint8_t> buffer() const noexcept + { + return {buffer_, buffer_size_}; + } + + //! \post `buffer.empty()`. + void clear_command() noexcept { buffer_size_ = 0; } + + //! Try to set `address` as remote connection request. + bool set_connect_command(const epee::net_utils::ipv4_network_address& address); + + //! Try to set `domain` + `port` as remote connection request. + bool set_connect_command(boost::string_ref domain, std::uint16_t port); + + //! Try to set `address` as remote Tor hidden service connection request. + bool set_connect_command(const net::tor_address& address); + + //! Try to set `address` as remote i2p hidden service connection request. + bool set_connect_command(const net::i2p_address& address); + + //! Try to set `domain` as remote DNS A record lookup request. + bool set_resolve_command(boost::string_ref domain); + + /*! + Asynchronously connect to `proxy_address` then issue command in + `buffer()`. The `done(...)` method will be invoked upon completion + with `self` and potential `error`s. + + \note Must use one of the `self->set_*_command` calls before using + this function. + + \param self ownership of object is given to function. + \param proxy_address of the socks server. + \return False if `self->buffer().empty()` (no command set). + */ + static bool connect_and_send(std::shared_ptr<client> self, const stream_type::endpoint& proxy_address); + + /*! + Assume existing connection to proxy server; asynchronously issue + command in `buffer()`. The `done(...)` method will be invoked + upon completion with `self` and potential `error`s. + + \note Must use one of the `self->set_*_command` calls before using + the function. + + \param self ownership of object is given to function. + \return False if `self->buffer().empty()` (no command set). + */ + static bool send(std::shared_ptr<client> self); + }; + + template<typename Handler> + class connect_client : public client + { + Handler handler_; + + virtual void done(boost::system::error_code error, std::shared_ptr<client>) override + { + handler_(error, take_socket()); + } + + public: + explicit connect_client(stream_type::socket&& proxy, socks::version ver, Handler&& handler) + : client(std::move(proxy), ver), handler_(std::move(handler)) + {} + + virtual ~connect_client() override {} + }; + + template<typename Handler> + inline std::shared_ptr<client> + make_connect_client(client::stream_type::socket&& proxy, socks::version ver, Handler handler) + { + return std::make_shared<connect_client<Handler>>(std::move(proxy), ver, std::move(handler)); + } +} // socks +} // net + +namespace boost +{ +namespace system +{ + template<> + struct is_error_code_enum<net::socks::error> + : true_type + {}; +} // system +} // boost diff --git a/src/net/tor_address.cpp b/src/net/tor_address.cpp new file mode 100644 index 000000000..904a9a0fc --- /dev/null +++ b/src/net/tor_address.cpp @@ -0,0 +1,203 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "tor_address.h" + +#include <algorithm> +#include <boost/spirit/include/karma_generate.hpp> +#include <boost/spirit/include/karma_uint.hpp> +#include <cassert> +#include <cstring> +#include <limits> + +#include "net/error.h" +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage.h" +#include "string_tools.h" + +namespace net +{ + namespace + { + constexpr const char tld[] = u8".onion"; + constexpr const char unknown_host[] = "<unknown tor host>"; + + constexpr const unsigned v2_length = 16; + constexpr const unsigned v3_length = 56; + + constexpr const char base32_alphabet[] = + u8"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz234567"; + + expect<void> host_check(boost::string_ref host) noexcept + { + if (!host.ends_with(tld)) + return {net::error::expected_tld}; + + host.remove_suffix(sizeof(tld) - 1); + + //! \TODO v3 has checksum, base32 decoding is required to verify it + if (host.size() != v2_length && host.size() != v3_length) + return {net::error::invalid_tor_address}; + if (host.find_first_not_of(base32_alphabet) != boost::string_ref::npos) + return {net::error::invalid_tor_address}; + + return success(); + } + + struct tor_serialized + { + std::string host; + std::uint16_t port; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(host) + KV_SERIALIZE(port) + END_KV_SERIALIZE_MAP() + }; + } + + tor_address::tor_address(const boost::string_ref host, const std::uint16_t port) noexcept + : port_(port) + { + // this is a private constructor, throw if moved to public + assert(host.size() < sizeof(host_)); + + const std::size_t length = std::min(sizeof(host_) - 1, host.size()); + std::memcpy(host_, host.data(), length); + std::memset(host_ + length, 0, sizeof(host_) - length); + } + + const char* tor_address::unknown_str() noexcept + { + return unknown_host; + } + + tor_address::tor_address() noexcept + : port_(0) + { + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); + std::memset(host_ + sizeof(unknown_host), 0, sizeof(host_) - sizeof(unknown_host)); + } + + expect<tor_address> tor_address::make(const boost::string_ref address, const std::uint16_t default_port) + { + boost::string_ref host = address.substr(0, address.rfind(':')); + const boost::string_ref port = + address.substr(host.size() + (host.size() == address.size() ? 0 : 1)); + + MONERO_CHECK(host_check(host)); + + std::uint16_t porti = default_port; + if (!port.empty() && !epee::string_tools::get_xtype_from_string(porti, std::string{port})) + return {net::error::invalid_port}; + + static_assert(v2_length <= v3_length, "bad internal host size"); + static_assert(v3_length + sizeof(tld) == sizeof(tor_address::host_), "bad internal host size"); + return tor_address{host, porti}; + } + + bool tor_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent) + { + tor_serialized in{}; + if (in._load(src, hparent) && in.host.size() < sizeof(host_) && (in.host == unknown_host || !host_check(in.host).has_error())) + { + std::memcpy(host_, in.host.data(), in.host.size()); + std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size()); + port_ = in.port; + return true; + } + static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size"); + std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator + port_ = 0; + return false; + } + + bool tor_address::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const + { + const tor_serialized out{std::string{host_}, port_}; + return out.store(dest, hparent); + } + + tor_address::tor_address(const tor_address& rhs) noexcept + : port_(rhs.port_) + { + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + + tor_address& tor_address::operator=(const tor_address& rhs) noexcept + { + if (this != std::addressof(rhs)) + { + port_ = rhs.port_; + std::memcpy(host_, rhs.host_, sizeof(host_)); + } + return *this; + } + + bool tor_address::is_unknown() const noexcept + { + static_assert(1 <= sizeof(host_), "host size too small"); + return host_[0] == '<'; // character is not allowed otherwise + } + + bool tor_address::equal(const tor_address& rhs) const noexcept + { + return port_ == rhs.port_ && is_same_host(rhs); + } + + bool tor_address::less(const tor_address& rhs) const noexcept + { + return std::strcmp(host_str(), rhs.host_str()) < 0 || port() < rhs.port(); + } + + bool tor_address::is_same_host(const tor_address& rhs) const noexcept + { + //! \TODO v2 and v3 should be comparable - requires base32 + return std::strcmp(host_str(), rhs.host_str()) == 0; + } + + std::string tor_address::str() const + { + const std::size_t host_length = std::strlen(host_str()); + const std::size_t port_length = + port_ == 0 ? 0 : std::numeric_limits<std::uint16_t>::digits10 + 2; + + std::string out{}; + out.reserve(host_length + port_length); + out.assign(host_str(), host_length); + + if (port_ != 0) + { + out.push_back(':'); + namespace karma = boost::spirit::karma; + karma::generate(std::back_inserter(out), karma::ushort_, port()); + } + return out; + } +} diff --git a/src/net/tor_address.h b/src/net/tor_address.h new file mode 100644 index 000000000..22d8cc119 --- /dev/null +++ b/src/net/tor_address.h @@ -0,0 +1,140 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/utility/string_ref.hpp> +#include <cstdint> +#include <string> + +#include "common/expect.h" +#include "net/enums.h" +#include "net/error.h" + +namespace epee +{ +namespace serialization +{ + class portable_storage; + struct section; +} +} + +namespace net +{ + //! Tor onion address; internal format not condensed/decoded. + class tor_address + { + std::uint16_t port_; + char host_[63]; // null-terminated + + //! Keep in private, `host.size()` has no runtime check + tor_address(boost::string_ref host, std::uint16_t port) noexcept; + + public: + //! \return Size of internal buffer for host. + static constexpr std::size_t buffer_size() noexcept { return sizeof(host_); } + + //! \return `<unknown tor host>`. + static const char* unknown_str() noexcept; + + //! An object with `port() == 0` and `host_str() == unknown_str()`. + tor_address() noexcept; + + //! \return A default constructed `tor_address` object. + static tor_address unknown() noexcept { return tor_address{}; } + + /*! + Parse `address` in onion v2 or v3 format with (i.e. x.onion:80) + with `default_port` being used iff port is not specified in + `address`. + */ + static expect<tor_address> make(boost::string_ref address, std::uint16_t default_port = 0); + + //! Load from epee p2p format, and \return false if not valid tor address + bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent); + + //! Store in epee p2p format + bool store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const; + + // Moves and copies are currently identical + + tor_address(const tor_address& rhs) noexcept; + ~tor_address() = default; + tor_address& operator=(const tor_address& rhs) noexcept; + + //! \return True if default constructed or via `unknown()`. + bool is_unknown() const noexcept; + + bool equal(const tor_address& rhs) const noexcept; + bool less(const tor_address& rhs) const noexcept; + + //! \return True if onion addresses are identical. + bool is_same_host(const tor_address& rhs) const noexcept; + + //! \return `x.onion` or `x.onion:z` if `port() != 0`. + std::string str() const; + + //! \return Null-terminated `x.onion` value or `unknown_str()`. + const char* host_str() const noexcept { return host_; } + + //! \return Port value or `0` if unspecified. + std::uint16_t port() const noexcept { return port_; } + + static constexpr bool is_loopback() noexcept { return false; } + static constexpr bool is_local() noexcept { return false; } + + static constexpr epee::net_utils::address_type get_type_id() noexcept + { + return epee::net_utils::address_type::tor; + } + + static constexpr epee::net_utils::zone get_zone() noexcept + { + return epee::net_utils::zone::tor; + } + + //! \return `!is_unknown()`. + bool is_blockable() const noexcept { return !is_unknown(); } + }; + + inline bool operator==(const tor_address& lhs, const tor_address& rhs) noexcept + { + return lhs.equal(rhs); + } + + inline bool operator!=(const tor_address& lhs, const tor_address& rhs) noexcept + { + return !lhs.equal(rhs); + } + + inline bool operator<(const tor_address& lhs, const tor_address& rhs) noexcept + { + return lhs.less(rhs); + } +} // net diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt index 9b924907e..3aecc3cf9 100644 --- a/src/p2p/CMakeLists.txt +++ b/src/p2p/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -40,6 +40,7 @@ target_link_libraries(p2p PUBLIC version cryptonote_core + net ${UPNP_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 7cad6e077..fcbcce58c 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -28,9 +28,83 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#include <boost/algorithm/string/find_iterator.hpp> +#include <boost/algorithm/string/finder.hpp> +#include <boost/chrono/duration.hpp> +#include <boost/endian/conversion.hpp> +#include <boost/optional/optional.hpp> +#include <boost/thread/future.hpp> +#include <boost/utility/string_ref.hpp> +#include <chrono> +#include <utility> + #include "common/command_line.h" #include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "net_node.h" +#include "net/net_utils_base.h" +#include "net/socks.h" +#include "net/parse.h" +#include "net/tor_address.h" +#include "net/i2p_address.h" +#include "p2p/p2p_protocol_defs.h" +#include "string_tools.h" + +namespace +{ + constexpr const boost::chrono::milliseconds future_poll_interval{500}; + constexpr const std::chrono::seconds socks_connect_timeout{P2P_DEFAULT_SOCKS_CONNECT_TIMEOUT}; + + std::int64_t get_max_connections(const boost::iterator_range<boost::string_ref::const_iterator> value) noexcept + { + // -1 is default, 0 is error + if (value.empty()) + return -1; + + std::uint32_t out = 0; + if (epee::string_tools::get_xtype_from_string(out, std::string{value.begin(), value.end()})) + return out; + return 0; + } + + template<typename T> + epee::net_utils::network_address get_address(const boost::string_ref value) + { + expect<T> address = T::make(value); + if (!address) + { + MERROR( + "Failed to parse " << epee::net_utils::zone_to_string(T::get_zone()) << " address \"" << value << "\": " << address.error().message() + ); + return {}; + } + return {std::move(*address)}; + } + + bool start_socks(std::shared_ptr<net::socks::client> client, const boost::asio::ip::tcp::endpoint& proxy, const epee::net_utils::network_address& remote) + { + CHECK_AND_ASSERT_MES(client != nullptr, false, "Unexpected null client"); + + bool set = false; + switch (remote.get_type_id()) + { + case net::tor_address::get_type_id(): + set = client->set_connect_command(remote.as<net::tor_address>()); + break; + case net::i2p_address::get_type_id(): + set = client->set_connect_command(remote.as<net::i2p_address>()); + break; + default: + MERROR("Unsupported network address in socks_connect"); + return false; + } + + const bool sent = + set && net::socks::client::connect_and_send(std::move(client), proxy); + CHECK_AND_ASSERT_MES(sent, false, "Unexpected failure to init socks client"); + return true; + } +} namespace nodetool { @@ -55,7 +129,10 @@ namespace nodetool const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_add_exclusive_node = {"add-exclusive-node", "Specify list of peers to connect to only." " If this option is given the options add-priority-node and seed-node are ignored"}; const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; + const command_line::arg_descriptor<std::vector<std::string> > arg_proxy = {"proxy", "<network-type>,<socks-ip:port>[,max_connections] i.e. \"tor,127.0.0.1:9050,100\""}; + const command_line::arg_descriptor<std::vector<std::string> > arg_anonymous_inbound = {"anonymous-inbound", "<hidden-service-address>,<[bind-ip:]port>[,max_connections] i.e. \"x.onion,127.0.0.1:18083,100\""}; const command_line::arg_descriptor<bool> arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; + const command_line::arg_descriptor<bool> arg_no_sync = {"no-sync", "Don't synchronize the blockchain with other peers", false}; const command_line::arg_descriptor<bool> arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; const command_line::arg_descriptor<int64_t> arg_out_peers = {"out-peers", "set max number of out peers", -1}; @@ -67,4 +144,203 @@ namespace nodetool const command_line::arg_descriptor<int64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", -1}; const command_line::arg_descriptor<bool> arg_save_graph = {"save-graph", "Save data for dr monero", false}; + + boost::optional<std::vector<proxy>> get_proxies(boost::program_options::variables_map const& vm) + { + namespace ip = boost::asio::ip; + + std::vector<proxy> proxies{}; + + const std::vector<std::string> args = command_line::get_arg(vm, arg_proxy); + proxies.reserve(args.size()); + + for (const boost::string_ref arg : args) + { + proxies.emplace_back(); + + auto next = boost::algorithm::make_split_iterator(arg, boost::algorithm::first_finder(",")); + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No network type for --" << arg_proxy.name); + const boost::string_ref zone{next->begin(), next->size()}; + + ++next; + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No ipv4:port given for --" << arg_proxy.name); + const boost::string_ref proxy{next->begin(), next->size()}; + + ++next; + if (!next.eof()) + { + proxies.back().max_connections = get_max_connections(*next); + if (proxies.back().max_connections == 0) + { + MERROR("Invalid max connections given to --" << arg_proxy.name); + return boost::none; + } + } + + switch (epee::net_utils::zone_from_string(zone)) + { + case epee::net_utils::zone::tor: + proxies.back().zone = epee::net_utils::zone::tor; + break; + case epee::net_utils::zone::i2p: + proxies.back().zone = epee::net_utils::zone::i2p; + break; + default: + MERROR("Invalid network for --" << arg_proxy.name); + return boost::none; + } + + std::uint32_t ip = 0; + std::uint16_t port = 0; + if (!epee::string_tools::parse_peer_from_string(ip, port, std::string{proxy}) || port == 0) + { + MERROR("Invalid ipv4:port given for --" << arg_proxy.name); + return boost::none; + } + proxies.back().address = ip::tcp::endpoint{ip::address_v4{boost::endian::native_to_big(ip)}, port}; + } + + return proxies; + } + + boost::optional<std::vector<anonymous_inbound>> get_anonymous_inbounds(boost::program_options::variables_map const& vm) + { + std::vector<anonymous_inbound> inbounds{}; + + const std::vector<std::string> args = command_line::get_arg(vm, arg_anonymous_inbound); + inbounds.reserve(args.size()); + + for (const boost::string_ref arg : args) + { + inbounds.emplace_back(); + + auto next = boost::algorithm::make_split_iterator(arg, boost::algorithm::first_finder(",")); + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No inbound address for --" << arg_anonymous_inbound.name); + const boost::string_ref address{next->begin(), next->size()}; + + ++next; + CHECK_AND_ASSERT_MES(!next.eof() && !next->empty(), boost::none, "No local ipv4:port given for --" << arg_anonymous_inbound.name); + const boost::string_ref bind{next->begin(), next->size()}; + + const std::size_t colon = bind.find_first_of(':'); + CHECK_AND_ASSERT_MES(colon < bind.size(), boost::none, "No local port given for --" << arg_anonymous_inbound.name); + + ++next; + if (!next.eof()) + { + inbounds.back().max_connections = get_max_connections(*next); + if (inbounds.back().max_connections == 0) + { + MERROR("Invalid max connections given to --" << arg_proxy.name); + return boost::none; + } + } + + expect<epee::net_utils::network_address> our_address = net::get_network_address(address, 0); + switch (our_address ? our_address->get_type_id() : epee::net_utils::address_type::invalid) + { + case net::tor_address::get_type_id(): + inbounds.back().our_address = std::move(*our_address); + inbounds.back().default_remote = net::tor_address::unknown(); + break; + case net::i2p_address::get_type_id(): + inbounds.back().our_address = std::move(*our_address); + inbounds.back().default_remote = net::i2p_address::unknown(); + break; + default: + MERROR("Invalid inbound address (" << address << ") for --" << arg_anonymous_inbound.name << ": " << (our_address ? "invalid type" : our_address.error().message())); + return boost::none; + } + + // get_address returns default constructed address on error + if (inbounds.back().our_address == epee::net_utils::network_address{}) + return boost::none; + + std::uint32_t ip = 0; + std::uint16_t port = 0; + if (!epee::string_tools::parse_peer_from_string(ip, port, std::string{bind})) + { + MERROR("Invalid ipv4:port given for --" << arg_anonymous_inbound.name); + return boost::none; + } + inbounds.back().local_ip = std::string{bind.substr(0, colon)}; + inbounds.back().local_port = std::string{bind.substr(colon + 1)}; + } + + return inbounds; + } + + bool is_filtered_command(const epee::net_utils::network_address& address, int command) + { + switch (command) + { + case nodetool::COMMAND_HANDSHAKE_T<cryptonote::CORE_SYNC_DATA>::ID: + case nodetool::COMMAND_TIMED_SYNC_T<cryptonote::CORE_SYNC_DATA>::ID: + case cryptonote::NOTIFY_NEW_TRANSACTIONS::ID: + return false; + default: + break; + } + + if (address.get_zone() == epee::net_utils::zone::public_) + return false; + + MWARNING("Filtered command (#" << command << ") to/from " << address.str()); + return true; + } + + boost::optional<boost::asio::ip::tcp::socket> + socks_connect_internal(const std::atomic<bool>& stop_signal, boost::asio::io_service& service, const boost::asio::ip::tcp::endpoint& proxy, const epee::net_utils::network_address& remote) + { + using socket_type = net::socks::client::stream_type::socket; + using client_result = std::pair<boost::system::error_code, socket_type>; + + struct notify + { + boost::promise<client_result> socks_promise; + + void operator()(boost::system::error_code error, socket_type&& sock) + { + socks_promise.set_value(std::make_pair(error, std::move(sock))); + } + }; + + boost::unique_future<client_result> socks_result{}; + { + boost::promise<client_result> socks_promise{}; + socks_result = socks_promise.get_future(); + + auto client = net::socks::make_connect_client( + boost::asio::ip::tcp::socket{service}, net::socks::version::v4a, notify{std::move(socks_promise)} + ); + if (!start_socks(std::move(client), proxy, remote)) + return boost::none; + } + + const auto start = std::chrono::steady_clock::now(); + while (socks_result.wait_for(future_poll_interval) == boost::future_status::timeout) + { + if (socks_connect_timeout < std::chrono::steady_clock::now() - start) + { + MERROR("Timeout on socks connect (" << proxy << " to " << remote.str() << ")"); + return boost::none; + } + + if (stop_signal) + return boost::none; + } + + try + { + auto result = socks_result.get(); + if (!result.first) + return {std::move(result.second)}; + + MERROR("Failed to make socks connection to " << remote.str() << " (via " << proxy << "): " << result.first.message()); + } + catch (boost::broken_promise const&) + {} + + return boost::none; + } } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 4db0a6cb7..42bb3b061 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -29,12 +29,18 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once +#include <array> +#include <atomic> +#include <boost/asio/io_service.hpp> +#include <boost/asio/ip/tcp.hpp> #include <boost/thread.hpp> +#include <boost/optional/optional_fwd.hpp> #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> -#include <boost/serialization/version.hpp> #include <boost/uuid/uuid.hpp> -#include <boost/serialization/map.hpp> +#include <functional> +#include <utility> +#include <vector> #include "cryptonote_config.h" #include "warnings.h" @@ -46,6 +52,8 @@ #include "net_peerlist.h" #include "math_helper.h" #include "net_node_common.h" +#include "net/enums.h" +#include "net/fwd.h" #include "common/command_line.h" PUSH_WARNINGS @@ -53,6 +61,47 @@ DISABLE_VS_WARNINGS(4355) namespace nodetool { + struct proxy + { + proxy() + : max_connections(-1), + address(), + zone(epee::net_utils::zone::invalid) + {} + + std::int64_t max_connections; + boost::asio::ip::tcp::endpoint address; + epee::net_utils::zone zone; + }; + + struct anonymous_inbound + { + anonymous_inbound() + : max_connections(-1), + local_ip(), + local_port(), + our_address(), + default_remote() + {} + + std::int64_t max_connections; + std::string local_ip; + std::string local_port; + epee::net_utils::network_address our_address; + epee::net_utils::network_address default_remote; + }; + + boost::optional<std::vector<proxy>> get_proxies(const boost::program_options::variables_map& vm); + boost::optional<std::vector<anonymous_inbound>> get_anonymous_inbounds(const boost::program_options::variables_map& vm); + + //! \return True if `commnd` is filtered (ignored/dropped) for `address` + bool is_filtered_command(epee::net_utils::network_address const& address, int command); + + // hides boost::future and chrono stuff from mondo template file + boost::optional<boost::asio::ip::tcp::socket> + socks_connect_internal(const std::atomic<bool>& stop_signal, boost::asio::io_service& service, const boost::asio::ip::tcp::endpoint& proxy, const epee::net_utils::network_address& remote); + + template<class base_type> struct p2p_connection_context_t: base_type //t_payload_net_handler::connection_context //public net_utils::connection_context_base { @@ -77,56 +126,134 @@ namespace nodetool typedef COMMAND_HANDSHAKE_T<typename t_payload_net_handler::payload_type> COMMAND_HANDSHAKE; typedef COMMAND_TIMED_SYNC_T<typename t_payload_net_handler::payload_type> COMMAND_TIMED_SYNC; + typedef epee::net_utils::boosted_tcp_server<epee::levin::async_protocol_handler<p2p_connection_context>> net_server; + + struct network_zone; + using connect_func = boost::optional<p2p_connection_context>(network_zone&, epee::net_utils::network_address const&, epee::net_utils::ssl_support_t); + + struct config_t + { + config_t() + : m_net_config(), + m_peer_id(crypto::rand<uint64_t>()), + m_support_flags(0) + {} + + network_config m_net_config; + uint64_t m_peer_id; + uint32_t m_support_flags; + }; + typedef epee::misc_utils::struct_init<config_t> config; + + struct network_zone + { + network_zone() + : m_connect(nullptr), + m_net_server(epee::net_utils::e_connection_type_P2P), + m_bind_ip(), + m_port(), + m_our_address(), + m_peerlist(), + m_config{}, + m_proxy_address(), + m_current_number_of_out_peers(0), + m_current_number_of_in_peers(0), + m_can_pingback(false) + { + set_config_defaults(); + } + + network_zone(boost::asio::io_service& public_service) + : m_connect(nullptr), + m_net_server(public_service, epee::net_utils::e_connection_type_P2P), + m_bind_ip(), + m_port(), + m_our_address(), + m_peerlist(), + m_config{}, + m_proxy_address(), + m_current_number_of_out_peers(0), + m_current_number_of_in_peers(0), + m_can_pingback(false) + { + set_config_defaults(); + } + + connect_func* m_connect; + net_server m_net_server; + std::string m_bind_ip; + std::string m_port; + epee::net_utils::network_address m_our_address; // in anonymity networks + peerlist_manager m_peerlist; + config m_config; + boost::asio::ip::tcp::endpoint m_proxy_address; + std::atomic<unsigned int> m_current_number_of_out_peers; + std::atomic<unsigned int> m_current_number_of_in_peers; + bool m_can_pingback; + + private: + void set_config_defaults() noexcept + { + // at this moment we have a hardcoded config + m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL; + m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; + m_config.m_net_config.config_id = 0; + m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT; + m_config.m_net_config.ping_connection_timeout = P2P_DEFAULT_PING_CONNECTION_TIMEOUT; + m_config.m_net_config.send_peerlist_sz = P2P_DEFAULT_PEERS_IN_HANDSHAKE; + m_config.m_support_flags = 0; // only set in public zone + } + }; + public: typedef t_payload_net_handler payload_net_handler; node_server(t_payload_net_handler& payload_handler) - :m_payload_handler(payload_handler), - m_current_number_of_out_peers(0), - m_current_number_of_in_peers(0), - m_allow_local_ip(false), - m_hide_my_port(false), - m_no_igd(false), - m_offline(false), - m_save_graph(false), - is_closing(false), - m_net_server( epee::net_utils::e_connection_type_P2P ) // this is a P2P connection of the main p2p node server, because this is class node_server<> - {} - virtual ~node_server() + : m_payload_handler(payload_handler), + m_external_port(0), + m_rpc_port(0), + m_allow_local_ip(false), + m_hide_my_port(false), + m_no_igd(false), + m_offline(false), + m_save_graph(false), + is_closing(false), + m_network_id() {} + virtual ~node_server(); static void init_options(boost::program_options::options_description& desc); bool run(); + network_zone& add_zone(epee::net_utils::zone zone); bool init(const boost::program_options::variables_map& vm); bool deinit(); bool send_stop_signal(); uint32_t get_this_peer_port(){return m_listening_port;} t_payload_net_handler& get_payload_object(); - template <class Archive, class t_version_type> - void serialize(Archive &a, const t_version_type ver) - { - a & m_peerlist; - if (ver == 0) - { - // from v1, we do not store the peer id anymore - peerid_type peer_id = AUTO_VAL_INIT (peer_id); - a & peer_id; - } - } // debug functions bool log_peerlist(); bool log_connections(); - virtual uint64_t get_connections_count(); - size_t get_outgoing_connections_count(); - size_t get_incoming_connections_count(); - peerlist_manager& get_peerlist_manager(){return m_peerlist;} - void delete_out_connections(size_t count); - void delete_in_connections(size_t count); + + // These functions only return information for the "public" zone + virtual uint64_t get_public_connections_count(); + size_t get_public_outgoing_connections_count(); + size_t get_public_white_peers_count(); + size_t get_public_gray_peers_count(); + void get_public_peerlist(std::vector<peerlist_entry>& gray, std::vector<peerlist_entry>& white); + size_t get_zone_count() const { return m_network_zones.size(); } + + void change_max_out_public_peers(size_t count); + void change_max_in_public_peers(size_t count); virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME); virtual bool unblock_host(const epee::net_utils::network_address &address); virtual std::map<std::string, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + + virtual void add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void clear_used_stripe_peers(); + private: const std::vector<std::string> m_seed_nodes_list = { "seeds.moneroseeds.se" @@ -144,6 +271,9 @@ namespace nodetool CHAIN_LEVIN_NOTIFY_MAP2(p2p_connection_context); //move levin_commands_handler interface notify(...) callbacks into nothing BEGIN_INVOKE_MAP2(node_server) + if (is_filtered_command(context.m_remote_address, command)) + return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; + HANDLE_INVOKE_T2(COMMAND_HANDSHAKE, &node_server::handle_handshake) HANDLE_INVOKE_T2(COMMAND_TIMED_SYNC, &node_server::handle_timed_sync) HANDLE_INVOKE_T2(COMMAND_PING, &node_server::handle_ping) @@ -172,7 +302,7 @@ namespace nodetool bool make_default_peer_id(); bool make_default_config(); bool store_config(); - bool check_trust(const proof_of_trust& tr); + bool check_trust(const proof_of_trust& tr, epee::net_utils::zone zone_type); //----------------- levin_commands_handler ------------------------------------------------------------- @@ -180,10 +310,9 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid> &connections); - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context); + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); @@ -198,7 +327,7 @@ namespace nodetool ); bool idle_worker(); bool handle_remote_peerlist(const std::vector<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context); - bool get_local_node_data(basic_node_data& node_data); + bool get_local_node_data(basic_node_data& node_data, const network_zone& zone); //bool get_local_handshake_data(handshake_data& hshd); bool merge_peerlist_with_local(const std::vector<peerlist_entry>& bs); @@ -210,7 +339,7 @@ namespace nodetool bool do_peer_timed_sync(const epee::net_utils::connection_context_base& context, peerid_type peer_id); bool make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist); - bool make_new_connection_from_peerlist(bool use_white_list); + bool make_new_connection_from_peerlist(network_zone& zone, bool use_white_list); bool try_to_connect_and_handshake_with_new_peer(const epee::net_utils::network_address& na, bool just_take_peerlist = false, uint64_t last_seen_stamp = 0, PeerType peer_type = white, uint64_t first_seen_stamp = 0); size_t get_random_index_with_fixed_probability(size_t max_index); bool is_peer_used(const peerlist_entry& peer); @@ -221,7 +350,7 @@ namespace nodetool template<class t_callback> bool try_ping(basic_node_data& node_data, p2p_connection_context& context, const t_callback &cb); bool try_get_support_flags(const p2p_connection_context& context, std::function<void(p2p_connection_context&, const uint32_t&)> f); - bool make_expected_connections_count(PeerType peer_type, size_t expected_connections); + bool make_expected_connections_count(network_zone& zone, PeerType peer_type, size_t expected_connections); void cache_connect_fail_info(const epee::net_utils::network_address& addr); bool is_addr_recently_failed(const epee::net_utils::network_address& addr); bool is_priority_node(const epee::net_utils::network_address& na); @@ -234,8 +363,8 @@ namespace nodetool template <class Container> bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container); - bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max); - bool set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max); + bool set_max_out_peers(network_zone& zone, int64_t max); + bool set_max_in_peers(network_zone& zone, int64_t max); bool set_tos_flag(const boost::program_options::variables_map& vm, int limit); bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit); @@ -243,6 +372,11 @@ namespace nodetool bool set_rate_limit(const boost::program_options::variables_map& vm, int64_t limit); bool has_too_many_connections(const epee::net_utils::network_address &address); + uint64_t get_connections_count(); + size_t get_incoming_connections_count(); + size_t get_incoming_connections_count(network_zone&); + size_t get_outgoing_connections_count(); + size_t get_outgoing_connections_count(network_zone&); bool check_connection_and_handshake_with_peer(const epee::net_utils::network_address& na, uint64_t last_seen_stamp); bool gray_peerlist_housekeeping(); @@ -260,31 +394,19 @@ namespace nodetool std::string print_connections_container(); - typedef epee::net_utils::boosted_tcp_server<epee::levin::async_protocol_handler<p2p_connection_context> > net_server; - - struct config - { - network_config m_net_config; - uint64_t m_peer_id; - uint32_t m_support_flags; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(m_net_config) - KV_SERIALIZE(m_peer_id) - KV_SERIALIZE(m_support_flags) - END_KV_SERIALIZE_MAP() - }; - public: - config m_config; // TODO was private, add getters? - std::atomic<unsigned int> m_current_number_of_out_peers; - std::atomic<unsigned int> m_current_number_of_in_peers; void set_save_graph(bool save_graph) { m_save_graph = save_graph; epee::net_utils::connection_basic::set_save_graph(save_graph); } + + void set_rpc_port(uint16_t rpc_port) + { + m_rpc_port = rpc_port; + } + private: std::string m_config_folder; @@ -292,7 +414,7 @@ namespace nodetool bool m_first_connection_maker_call; uint32_t m_listening_port; uint32_t m_external_port; - uint32_t m_ip_address; + uint16_t m_rpc_port; bool m_allow_local_ip; bool m_hide_my_port; bool m_no_igd; @@ -304,7 +426,7 @@ namespace nodetool //connections_indexed_container m_connections; t_payload_net_handler& m_payload_handler; - peerlist_manager m_peerlist; + peerlist_storage m_peerlist_storage; epee::math_helper::once_a_time_seconds<P2P_DEFAULT_HANDSHAKE_INTERVAL> m_peer_handshake_idle_maker_interval; epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval; @@ -312,8 +434,6 @@ namespace nodetool epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval; epee::math_helper::once_a_time_seconds<3600, false> m_incoming_connections_interval; - std::string m_bind_ip; - std::string m_port; #ifdef ALLOW_DEBUG_COMMANDS uint64_t m_last_stat_request_time; #endif @@ -321,11 +441,22 @@ namespace nodetool std::vector<epee::net_utils::network_address> m_exclusive_peers; std::vector<epee::net_utils::network_address> m_seed_nodes; bool m_fallback_seed_nodes_added; - std::list<nodetool::peerlist_entry> m_command_line_peers; + std::vector<nodetool::peerlist_entry> m_command_line_peers; uint64_t m_peer_livetime; //keep connections to initiate some interactions - net_server m_net_server; - boost::uuids::uuid m_network_id; + + + static boost::optional<p2p_connection_context> public_connect(network_zone&, epee::net_utils::network_address const&, epee::net_utils::ssl_support_t); + static boost::optional<p2p_connection_context> socks_connect(network_zone&, epee::net_utils::network_address const&, epee::net_utils::ssl_support_t); + + + /* A `std::map` provides constant iterators and key/value pointers even with + inserts/erases to _other_ elements. This makes the configuration step easier + since references can safely be stored on the stack. Do not insert/erase + after configuration and before destruction, lock safety would need to be + added. `std::map::operator[]` WILL insert! */ + std::map<epee::net_utils::zone, network_zone> m_network_zones; + std::map<epee::net_utils::network_address, time_t> m_conn_fails_cache; epee::critical_section m_conn_fails_cache_lock; @@ -336,7 +467,13 @@ namespace nodetool epee::critical_section m_host_fails_score_lock; std::map<std::string, uint64_t> m_host_fails_score; + boost::mutex m_used_stripe_peers_mutex; + std::array<std::list<epee::net_utils::network_address>, 1 << CRYPTONOTE_PRUNING_LOG_STRIPES> m_used_stripe_peers; + + boost::uuids::uuid m_network_id; cryptonote::network_type m_nettype; + + epee::net_utils::ssl_support_t m_ssl_support; }; const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s @@ -349,7 +486,10 @@ namespace nodetool extern const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_add_priority_node; extern const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_add_exclusive_node; extern const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node; + extern const command_line::arg_descriptor<std::vector<std::string> > arg_proxy; + extern const command_line::arg_descriptor<std::vector<std::string> > arg_anonymous_inbound; extern const command_line::arg_descriptor<bool> arg_p2p_hide_my_port; + extern const command_line::arg_descriptor<bool> arg_no_sync; extern const command_line::arg_descriptor<bool> arg_no_igd; extern const command_line::arg_descriptor<bool> arg_offline; @@ -365,3 +505,4 @@ namespace nodetool } POP_WARNINGS + diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 5b845fe15..ba6e79d3f 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,23 +31,34 @@ // IP blocking adapted from Boolberry #include <algorithm> +#include <boost/bind.hpp> #include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/filesystem/operations.hpp> +#include <boost/optional/optional.hpp> #include <boost/thread/thread.hpp> -#include <boost/bind.hpp> +#include <boost/uuid/uuid_io.hpp> #include <atomic> +#include <functional> +#include <limits> +#include <memory> +#include <tuple> +#include <vector> #include "version.h" #include "string_tools.h" #include "common/util.h" #include "common/dns_utils.h" +#include "common/pruning.h" +#include "net/error.h" #include "net/net_helper.h" #include "math_helper.h" +#include "misc_log_ex.h" #include "p2p_protocol_defs.h" -#include "net_peerlist_boost_serialization.h" #include "net/local_ip.h" #include "crypto/crypto.h" #include "storages/levin_abstract_invoke2.h" #include "cryptonote_core/cryptonote_core.h" +#include "net/parse.h" #include <miniupnp/miniupnpc/miniupnpc.h> #include <miniupnp/miniupnpc/upnpcommands.h> @@ -62,6 +73,20 @@ namespace nodetool { + template<class t_payload_net_handler> + node_server<t_payload_net_handler>::~node_server() + { + // tcp server uses io_service in destructor, and every zone uses + // io_service from public zone. + for (auto current = m_network_zones.begin(); current != m_network_zones.end(); /* below */) + { + if (current->first != epee::net_utils::zone::public_) + current = m_network_zones.erase(current); + else + ++current; + } + } + //----------------------------------------------------------------------------------- inline bool append_net_address(std::vector<epee::net_utils::network_address> & seed_nodes, std::string const & addr, uint16_t default_port); //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -75,7 +100,10 @@ namespace nodetool command_line::add_arg(desc, arg_p2p_add_priority_node); command_line::add_arg(desc, arg_p2p_add_exclusive_node); command_line::add_arg(desc, arg_p2p_seed_node); + command_line::add_arg(desc, arg_proxy); + command_line::add_arg(desc, arg_anonymous_inbound); command_line::add_arg(desc, arg_p2p_hide_my_port); + command_line::add_arg(desc, arg_no_sync); command_line::add_arg(desc, arg_no_igd); command_line::add_arg(desc, arg_out_peers); command_line::add_arg(desc, arg_in_peers); @@ -89,62 +117,14 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::init_config() { - // TRY_ENTRY(); - std::string state_file_path = m_config_folder + "/" + P2P_NET_DATA_FILENAME; - std::ifstream p2p_data; - p2p_data.open( state_file_path , std::ios_base::binary | std::ios_base::in); - if(!p2p_data.fail()) - { - try - { - // first try reading in portable mode - boost::archive::portable_binary_iarchive a(p2p_data); - a >> *this; - } - catch (...) - { - // if failed, try reading in unportable mode - boost::filesystem::copy_file(state_file_path, state_file_path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); - p2p_data.close(); - p2p_data.open( state_file_path , std::ios_base::binary | std::ios_base::in); - if(!p2p_data.fail()) - { - try - { - boost::archive::binary_iarchive a(p2p_data); - a >> *this; - } - catch (const std::exception &e) - { - MWARNING("Failed to load p2p config file, falling back to default config"); - m_peerlist = peerlist_manager(); // it was probably half clobbered by the failed load - make_default_config(); - } - } - else - { - make_default_config(); - } - } - }else - { - make_default_config(); - } - - // always recreate a new peer id - make_default_peer_id(); - - //at this moment we have hardcoded config - m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL; - m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; //20 MB limit - m_config.m_net_config.config_id = 0; // initial config - m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT; - m_config.m_net_config.ping_connection_timeout = P2P_DEFAULT_PING_CONNECTION_TIMEOUT; - m_config.m_net_config.send_peerlist_sz = P2P_DEFAULT_PEERS_IN_HANDSHAKE; - m_config.m_support_flags = P2P_SUPPORT_FLAGS; + auto storage = peerlist_storage::open(m_config_folder + "/" + P2P_NET_DATA_FILENAME); + if (storage) + m_peerlist_storage = std::move(*storage); + m_network_zones[epee::net_utils::zone::public_].m_config.m_support_flags = P2P_SUPPORT_FLAGS; m_first_connection_maker_call = true; + CATCH_ENTRY_L0("node_server::init_config", false); return true; } @@ -152,17 +132,26 @@ namespace nodetool template<class t_payload_net_handler> void node_server<t_payload_net_handler>::for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f) { - m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntx){ - return f(cntx, cntx.peer_id, cntx.support_flags); - }); + for(auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntx){ + return f(cntx, cntx.peer_id, cntx.support_flags); + }); + } } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::for_connection(const boost::uuids::uuid &connection_id, std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f) { - return m_net_server.get_config_object().for_connection(connection_id, [&](p2p_connection_context& cntx){ - return f(cntx, cntx.peer_id, cntx.support_flags); - }); + for(auto& zone : m_network_zones) + { + const bool result = zone.second.m_net_server.get_config_object().for_connection(connection_id, [&](p2p_connection_context& cntx){ + return f(cntx, cntx.peer_id, cntx.support_flags); + }); + if (result) + return true; + } + return false; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -182,36 +171,33 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::make_default_peer_id() - { - m_config.m_peer_id = crypto::rand<uint64_t>(); - return true; - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::make_default_config() - { - return make_default_peer_id(); - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::block_host(const epee::net_utils::network_address &addr, time_t seconds) { + if(!addr.is_blockable()) + return false; + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); m_blocked_hosts[addr.host_str()] = time(nullptr) + seconds; - // drop any connection to that IP - std::list<boost::uuids::uuid> conns; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + // drop any connection to that address. This should only have to look into + // the zone related to the connection, but really make sure everything is + // swept ... + std::vector<boost::uuids::uuid> conns; + for(auto& zone : m_network_zones) { - if (cntxt.m_remote_address.is_same_host(addr)) + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - conns.push_back(cntxt.m_connection_id); - } - return true; - }); - for (const auto &c: conns) - m_net_server.get_config_object().close(c); + if (cntxt.m_remote_address.is_same_host(addr)) + { + conns.push_back(cntxt.m_connection_id); + } + return true; + }); + for (const auto &c: conns) + zone.second.m_net_server.get_config_object().close(c); + + conns.clear(); + } MCLOG_CYAN(el::Level::Info, "global", "Host " << addr.host_str() << " blocked."); return true; @@ -232,6 +218,9 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::add_host_fail(const epee::net_utils::network_address &address) { + if(!address.is_blockable()) + return false; + CRITICAL_REGION_LOCAL(m_host_fails_score_lock); uint64_t fails = ++m_host_fails_score[address.host_str()]; MDEBUG("Host " << address.host_str() << " fail score=" << fails); @@ -246,12 +235,6 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::parse_peer_from_string(epee::net_utils::network_address& pe, const std::string& node_addr, uint16_t default_port) - { - return epee::net_utils::create_network_address(pe, node_addr, default_port); - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::handle_command_line( const boost::program_options::variables_map& vm ) @@ -260,8 +243,11 @@ namespace nodetool bool stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); m_nettype = testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET; - m_bind_ip = command_line::get_arg(vm, arg_p2p_bind_ip); - m_port = command_line::get_arg(vm, arg_p2p_bind_port); + network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; + public_zone.m_connect = &public_connect; + public_zone.m_bind_ip = command_line::get_arg(vm, arg_p2p_bind_ip); + public_zone.m_port = command_line::get_arg(vm, arg_p2p_bind_port); + public_zone.m_can_pingback = true; m_external_port = command_line::get_arg(vm, arg_p2p_external_port); m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip); m_no_igd = command_line::get_arg(vm, arg_no_igd); @@ -275,14 +261,20 @@ namespace nodetool nodetool::peerlist_entry pe = AUTO_VAL_INIT(pe); pe.id = crypto::rand<uint64_t>(); const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; - bool r = parse_peer_from_string(pe.adr, pr_str, default_port); - if (r) + expect<epee::net_utils::network_address> adr = net::get_network_address(pr_str, default_port); + if (adr) { - m_command_line_peers.push_back(pe); + add_zone(adr->get_zone()); + pe.adr = std::move(*adr); + m_command_line_peers.push_back(std::move(pe)); continue; } + CHECK_AND_ASSERT_MES( + adr == net::error::unsupported_address, false, "Bad address (\"" << pr_str << "\"): " << adr.error().message() + ); + std::vector<epee::net_utils::network_address> resolved_addrs; - r = append_net_address(resolved_addrs, pr_str, default_port); + bool r = append_net_address(resolved_addrs, pr_str, default_port); CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); for (const epee::net_utils::network_address& addr : resolved_addrs) { @@ -319,10 +311,16 @@ namespace nodetool if(command_line::has_arg(vm, arg_p2p_hide_my_port)) m_hide_my_port = true; - if ( !set_max_out_peers(vm, command_line::get_arg(vm, arg_out_peers) ) ) + if (command_line::has_arg(vm, arg_no_sync)) + m_payload_handler.set_no_sync(true); + + if ( !set_max_out_peers(public_zone, command_line::get_arg(vm, arg_out_peers) ) ) return false; + else + m_payload_handler.set_max_out_peers(public_zone.m_config.m_net_config.max_out_connection_count); + - if ( !set_max_in_peers(vm, command_line::get_arg(vm, arg_in_peers) ) ) + if ( !set_max_in_peers(public_zone, command_line::get_arg(vm, arg_in_peers) ) ) return false; if ( !set_tos_flag(vm, command_line::get_arg(vm, arg_tos_flag) ) ) @@ -337,6 +335,58 @@ namespace nodetool if ( !set_rate_limit(vm, command_line::get_arg(vm, arg_limit_rate) ) ) return false; + + auto proxies = get_proxies(vm); + if (!proxies) + return false; + + for (auto& proxy : *proxies) + { + network_zone& zone = add_zone(proxy.zone); + if (zone.m_connect != nullptr) + { + MERROR("Listed --" << arg_proxy.name << " twice with " << epee::net_utils::zone_to_string(proxy.zone)); + return false; + } + zone.m_connect = &socks_connect; + zone.m_proxy_address = std::move(proxy.address); + + if (!set_max_out_peers(zone, proxy.max_connections)) + return false; + } + + for (const auto& zone : m_network_zones) + { + if (zone.second.m_connect == nullptr) + { + MERROR("Set outgoing peer for " << epee::net_utils::zone_to_string(zone.first) << " but did not set --" << arg_proxy.name); + return false; + } + } + + auto inbounds = get_anonymous_inbounds(vm); + if (!inbounds) + return false; + + for (auto& inbound : *inbounds) + { + network_zone& zone = add_zone(inbound.our_address.get_zone()); + + if (!zone.m_bind_ip.empty()) + { + MERROR("Listed --" << arg_anonymous_inbound.name << " twice with " << epee::net_utils::zone_to_string(inbound.our_address.get_zone()) << " network"); + return false; + } + + zone.m_bind_ip = std::move(inbound.local_ip); + zone.m_port = std::move(inbound.local_port); + zone.m_net_server.set_default_remote(std::move(inbound.default_remote)); + zone.m_our_address = std::move(inbound.our_address); + + if (!set_max_in_peers(zone, inbound.max_connections)) + return false; + } + return true; } //----------------------------------------------------------------------------------- @@ -418,7 +468,17 @@ namespace nodetool } return full_addrs; } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + typename node_server<t_payload_net_handler>::network_zone& node_server<t_payload_net_handler>::add_zone(const epee::net_utils::zone zone) + { + const auto zone_ = m_network_zones.lower_bound(zone); + if (zone_ != m_network_zones.end() && zone_->first == zone) + return zone_->second; + network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; + return m_network_zones.emplace_hint(zone_, std::piecewise_construct, std::make_tuple(zone), std::tie(public_zone.m_net_server.get_io_service()))->second; + } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm) @@ -535,45 +595,82 @@ namespace nodetool MDEBUG("Number of seed nodes: " << m_seed_nodes.size()); m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir); + network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); - if ((m_nettype == cryptonote::MAINNET && m_port != std::to_string(::config::P2P_DEFAULT_PORT)) - || (m_nettype == cryptonote::TESTNET && m_port != std::to_string(::config::testnet::P2P_DEFAULT_PORT)) - || (m_nettype == cryptonote::STAGENET && m_port != std::to_string(::config::stagenet::P2P_DEFAULT_PORT))) { - m_config_folder = m_config_folder + "/" + m_port; + if ((m_nettype == cryptonote::MAINNET && public_zone.m_port != std::to_string(::config::P2P_DEFAULT_PORT)) + || (m_nettype == cryptonote::TESTNET && public_zone.m_port != std::to_string(::config::testnet::P2P_DEFAULT_PORT)) + || (m_nettype == cryptonote::STAGENET && public_zone.m_port != std::to_string(::config::stagenet::P2P_DEFAULT_PORT))) { + m_config_folder = m_config_folder + "/" + public_zone.m_port; } res = init_config(); CHECK_AND_ASSERT_MES(res, false, "Failed to init config."); - res = m_peerlist.init(m_allow_local_ip); - CHECK_AND_ASSERT_MES(res, false, "Failed to init peerlist."); + for (auto& zone : m_network_zones) + { + res = zone.second.m_peerlist.init(m_peerlist_storage.take_zone(zone.first), m_allow_local_ip); + CHECK_AND_ASSERT_MES(res, false, "Failed to init peerlist."); + } + for(const auto& p: m_command_line_peers) + m_network_zones.at(p.adr.get_zone()).m_peerlist.append_with_peer_white(p); - for(auto& p: m_command_line_peers) - m_peerlist.append_with_peer_white(p); +// all peers are now setup +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + for (auto& zone : m_network_zones) + { + std::list<peerlist_entry> plw; + while (zone.second.m_peerlist.get_white_peers_count()) + { + plw.push_back(peerlist_entry()); + zone.second.m_peerlist.get_white_peer_by_index(plw.back(), 0); + zone.second.m_peerlist.remove_from_peer_white(plw.back()); + } + for (auto &e:plw) + zone.second.m_peerlist.append_with_peer_white(e); + + std::list<peerlist_entry> plg; + while (zone.second.m_peerlist.get_gray_peers_count()) + { + plg.push_back(peerlist_entry()); + zone.second.m_peerlist.get_gray_peer_by_index(plg.back(), 0); + zone.second.m_peerlist.remove_from_peer_gray(plg.back()); + } + for (auto &e:plg) + zone.second.m_peerlist.append_with_peer_gray(e); + } +#endif //only in case if we really sure that we have external visible ip m_have_address = true; - m_ip_address = 0; m_last_stat_request_time = 0; //configure self - m_net_server.set_threads_prefix("P2P"); - m_net_server.get_config_object().set_handler(this); - m_net_server.get_config_object().m_invoke_timeout = P2P_DEFAULT_INVOKE_TIMEOUT; - m_net_server.set_connection_filter(this); + + public_zone.m_net_server.set_threads_prefix("P2P"); // all zones use these threads/asio::io_service // from here onwards, it's online stuff if (m_offline) return res; //try to bind - MINFO("Binding on " << m_bind_ip << ":" << m_port); - res = m_net_server.init_server(m_port, m_bind_ip); - CHECK_AND_ASSERT_MES(res, false, "Failed to bind server"); + m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; + for (auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().set_handler(this); + zone.second.m_net_server.get_config_object().m_invoke_timeout = P2P_DEFAULT_INVOKE_TIMEOUT; + + if (!zone.second.m_bind_ip.empty()) + { + zone.second.m_net_server.set_connection_filter(this); + MINFO("Binding on " << zone.second.m_bind_ip << ":" << zone.second.m_port); + res = zone.second.m_net_server.init_server(zone.second.m_port, zone.second.m_bind_ip, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + CHECK_AND_ASSERT_MES(res, false, "Failed to bind server"); + } + } - m_listening_port = m_net_server.get_binded_port(); - MLOG_GREEN(el::Level::Info, "Net service bound to " << m_bind_ip << ":" << m_listening_port); + m_listening_port = public_zone.m_net_server.get_binded_port(); + MLOG_GREEN(el::Level::Info, "Net service bound to " << public_zone.m_bind_ip << ":" << m_listening_port); if(m_external_port) MDEBUG("External port defined as " << m_external_port); @@ -597,44 +694,45 @@ namespace nodetool mPeersLoggerThread.reset(new boost::thread([&]() { _note("Thread monitor number of peers - start"); - while (!is_closing && !m_net_server.is_stop_signal_sent()) + const network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); + while (!is_closing && !public_zone.m_net_server.is_stop_signal_sent()) { // main loop of thread //number_of_peers = m_net_server.get_config_object().get_connections_count(); - unsigned int number_of_in_peers = 0; - unsigned int number_of_out_peers = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + for (auto& zone : m_network_zones) { - if (cntxt.m_is_income) - { - ++number_of_in_peers; - } - else + unsigned int number_of_in_peers = 0; + unsigned int number_of_out_peers = 0; + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - ++number_of_out_peers; - } - return true; - }); // lambda - - m_current_number_of_in_peers = number_of_in_peers; - m_current_number_of_out_peers = number_of_out_peers; - + if (cntxt.m_is_income) + { + ++number_of_in_peers; + } + else + { + ++number_of_out_peers; + } + return true; + }); // lambda + zone.second.m_current_number_of_in_peers = number_of_in_peers; + zone.second.m_current_number_of_out_peers = number_of_out_peers; + } boost::this_thread::sleep_for(boost::chrono::seconds(1)); } // main loop of thread _note("Thread monitor number of peers - done"); })); // lambda + network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); + public_zone.m_net_server.add_idle_handler(boost::bind(&node_server<t_payload_net_handler>::idle_worker, this), 1000); + public_zone.m_net_server.add_idle_handler(boost::bind(&t_payload_net_handler::on_idle, &m_payload_handler), 1000); + //here you can set worker threads count int thrds_count = 10; - - m_net_server.add_idle_handler(boost::bind(&node_server<t_payload_net_handler>::idle_worker, this), 1000); - m_net_server.add_idle_handler(boost::bind(&t_payload_net_handler::on_idle, &m_payload_handler), 1000); - boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - //go to loop MINFO("Run net_service loop( " << thrds_count << " threads)..."); - if(!m_net_server.run_server(thrds_count, true, attrs)) + if(!public_zone.m_net_server.run_server(thrds_count, true, attrs)) { LOG_ERROR("Failed to run net tcp server!"); } @@ -642,23 +740,34 @@ namespace nodetool MINFO("net_service loop stopped."); return true; } - + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + uint64_t node_server<t_payload_net_handler>::get_public_connections_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return public_zone->second.m_net_server.get_config_object().get_connections_count(); + } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> uint64_t node_server<t_payload_net_handler>::get_connections_count() { - return m_net_server.get_config_object().get_connections_count(); + std::uint64_t count = 0; + for (auto& zone : m_network_zones) + count += zone.second.m_net_server.get_config_object().get_connections_count(); + return count; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::deinit() { kill(); - m_peerlist.deinit(); if (!m_offline) { - m_net_server.deinit_server(); + for(auto& zone : m_network_zones) + zone.second.m_net_server.deinit_server(); // remove UPnP port mapping if(!m_no_igd) delete_upnp_port_mapping(m_listening_port); @@ -669,28 +778,25 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::store_config() { - TRY_ENTRY(); + if (!tools::create_directories_if_necessary(m_config_folder)) { - MWARNING("Failed to create data directory: " << m_config_folder); + MWARNING("Failed to create data directory \"" << m_config_folder); return false; } - std::string state_file_path = m_config_folder + "/" + P2P_NET_DATA_FILENAME; - std::ofstream p2p_data; - p2p_data.open( state_file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); - if(p2p_data.fail()) + peerlist_types active{}; + for (auto& zone : m_network_zones) + zone.second.m_peerlist.get_peerlist(active); + + const std::string state_file_path = m_config_folder + "/" + P2P_NET_DATA_FILENAME; + if (!m_peerlist_storage.store(state_file_path, active)) { MWARNING("Failed to save config to file " << state_file_path); return false; - }; - - boost::archive::portable_binary_oarchive a(p2p_data); - a << *this; - return true; - CATCH_ENTRY_L0("blockchain_storage::save", false); - + } + CATCH_ENTRY_L0("node_server::store", false); return true; } //----------------------------------------------------------------------------------- @@ -698,37 +804,39 @@ namespace nodetool bool node_server<t_payload_net_handler>::send_stop_signal() { MDEBUG("[node] sending stop signal"); - m_net_server.send_stop_signal(); + for (auto& zone : m_network_zones) + zone.second.m_net_server.send_stop_signal(); MDEBUG("[node] Stop signal sent"); - std::list<boost::uuids::uuid> connection_ids; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - connection_ids.push_back(cntxt.m_connection_id); - return true; - }); - for (const auto &connection_id: connection_ids) - m_net_server.get_config_object().close(connection_id); - + for (auto& zone : m_network_zones) + { + std::list<boost::uuids::uuid> connection_ids; + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { + connection_ids.push_back(cntxt.m_connection_id); + return true; + }); + for (const auto &connection_id: connection_ids) + zone.second.m_net_server.get_config_object().close(connection_id); + } m_payload_handler.stop(); - return true; } //----------------------------------------------------------------------------------- - - template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::do_handshake_with_peer(peerid_type& pi, p2p_connection_context& context_, bool just_take_peerlist) { + network_zone& zone = m_network_zones.at(context_.m_remote_address.get_zone()); + typename COMMAND_HANDSHAKE::request arg; typename COMMAND_HANDSHAKE::response rsp; - get_local_node_data(arg.node_data); + get_local_node_data(arg.node_data, zone); m_payload_handler.get_payload_sync_data(arg.payload_data); epee::simple_event ev; std::atomic<bool> hsh_result(false); - bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_HANDSHAKE::response>(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, m_net_server.get_config_object(), - [this, &pi, &ev, &hsh_result, &just_take_peerlist](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) + bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_HANDSHAKE::response>(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, zone.m_net_server.get_config_object(), + [this, &pi, &ev, &hsh_result, &just_take_peerlist, &context_](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ev.raise();}); @@ -740,7 +848,7 @@ namespace nodetool if(rsp.node_data.network_id != m_network_id) { - LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << epee::string_tools::get_str_from_guid_a(rsp.node_data.network_id) << "), closing connection."); + LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << rsp.node_data.network_id << "), closing connection."); return; } @@ -761,19 +869,26 @@ namespace nodetool } pi = context.peer_id = rsp.node_data.peer_id; - m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address); + context.m_rpc_port = rsp.node_data.rpc_port; + m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port); - if(rsp.node_data.peer_id == m_config.m_peer_id) + // move + for (auto const& zone : m_network_zones) { - LOG_DEBUG_CC(context, "Connection to self detected, dropping connection"); - hsh_result = false; - return; + if(rsp.node_data.peer_id == zone.second.m_config.m_peer_id) + { + LOG_DEBUG_CC(context, "Connection to self detected, dropping connection"); + hsh_result = false; + return; + } } + LOG_INFO_CC(context, "New connection handshaked, pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE INVOKED OK"); }else { LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE(AND CLOSE) INVOKED OK"); } + context_ = context; }, P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT); if(r) @@ -784,7 +899,7 @@ namespace nodetool if(!hsh_result) { LOG_WARNING_CC(context_, "COMMAND_HANDSHAKE Failed"); - m_net_server.get_config_object().close(context_.m_connection_id); + m_network_zones.at(context_.m_remote_address.get_zone()).m_net_server.get_config_object().close(context_.m_connection_id); } else { @@ -803,7 +918,8 @@ namespace nodetool typename COMMAND_TIMED_SYNC::request arg = AUTO_VAL_INIT(arg); m_payload_handler.get_payload_sync_data(arg.payload_data); - bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_TIMED_SYNC::response>(context_.m_connection_id, COMMAND_TIMED_SYNC::ID, arg, m_net_server.get_config_object(), + network_zone& zone = m_network_zones.at(context_.m_remote_address.get_zone()); + bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_TIMED_SYNC::response>(context_.m_connection_id, COMMAND_TIMED_SYNC::ID, arg, zone.m_net_server.get_config_object(), [this](int code, const typename COMMAND_TIMED_SYNC::response& rsp, p2p_connection_context& context) { context.m_in_timedsync = false; @@ -816,11 +932,11 @@ namespace nodetool if(!handle_remote_peerlist(rsp.local_peerlist_new, rsp.local_time, context)) { LOG_WARNING_CC(context, "COMMAND_TIMED_SYNC: failed to handle_remote_peerlist(...), closing connection."); - m_net_server.get_config_object().close(context.m_connection_id ); + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); add_host_fail(context.m_remote_address); } if(!context.m_is_income) - m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address); + m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port); m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false); }); @@ -848,53 +964,61 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::is_peer_used(const peerlist_entry& peer) { - - if(m_config.m_peer_id == peer.id) - return true;//dont make connections to ourself + for(const auto& zone : m_network_zones) + if(zone.second.m_config.m_peer_id == peer.id) + return true;//dont make connections to ourself bool used = false; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + for(auto& zone : m_network_zones) { - if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - used = true; - return false;//stop enumerating - } - return true; - }); + if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) + { + used = true; + return false;//stop enumerating + } + return true; + }); - return used; + if(used) + return true; + } + return false; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::is_peer_used(const anchor_peerlist_entry& peer) { - if(m_config.m_peer_id == peer.id) { - return true;//dont make connections to ourself - } - - bool used = false; - - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) - { - if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) - { - used = true; - - return false;//stop enumerating + for(auto& zone : m_network_zones) { + if(zone.second.m_config.m_peer_id == peer.id) { + return true;//dont make connections to ourself } - - return true; - }); - - return used; + bool used = false; + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address)) + { + used = true; + return false;//stop enumerating + } + return true; + }); + if (used) + return true; + } + return false; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::is_addr_connected(const epee::net_utils::network_address& peer) { + const auto zone = m_network_zones.find(peer.get_zone()); + if (zone == m_network_zones.end()) + return false; + bool connected = false; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + zone->second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { if(!cntxt.m_is_income && peer == cntxt.m_remote_address) { @@ -919,46 +1043,44 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::try_to_connect_and_handshake_with_new_peer(const epee::net_utils::network_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, PeerType peer_type, uint64_t first_seen_stamp) { - if (m_current_number_of_out_peers == m_config.m_net_config.max_out_connection_count) // out peers limit + network_zone& zone = m_network_zones.at(na.get_zone()); + if (zone.m_connect == nullptr) // outgoing connections in zone not possible + return false; + + if (zone.m_current_number_of_out_peers == zone.m_config.m_net_config.max_out_connection_count) // out peers limit { return false; } - else if (m_current_number_of_out_peers > m_config.m_net_config.max_out_connection_count) + else if (zone.m_current_number_of_out_peers > zone.m_config.m_net_config.max_out_connection_count) { - m_net_server.get_config_object().del_out_connections(1); - m_current_number_of_out_peers --; // atomic variable, update time = 1s + zone.m_net_server.get_config_object().del_out_connections(1); + --(zone.m_current_number_of_out_peers); // atomic variable, update time = 1s return false; } + + MDEBUG("Connecting to " << na.str() << "(peer_type=" << peer_type << ", last_seen: " << (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never") << ")..."); - CHECK_AND_ASSERT_MES(na.get_type_id() == epee::net_utils::ipv4_network_address::ID, false, - "Only IPv4 addresses are supported here"); - const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); - - typename net_server::t_connection_context con = AUTO_VAL_INIT(con); - bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), - epee::string_tools::num_to_string_fast(ipv4.port()), - m_config.m_net_config.connection_timeout, - con); - - if(!res) + auto con = zone.m_connect(zone, na, m_ssl_support); + if(!con) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Connect failed to " << na.str() + LOG_PRINT_CC_PRIORITY_NODE(is_priority, bool(con), "Connect failed to " << na.str() /*<< ", try " << try_count*/); //m_peerlist.set_peer_unreachable(pe); return false; } + con->m_anchor = peer_type == anchor; peerid_type pi = AUTO_VAL_INIT(pi); - res = do_handshake_with_peer(pi, con, just_take_peerlist); + bool res = do_handshake_with_peer(pi, *con, just_take_peerlist); if(!res) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Failed to HANDSHAKE with peer " + LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str() /*<< ", try " << try_count*/); return false; @@ -966,8 +1088,8 @@ namespace nodetool if(just_take_peerlist) { - m_net_server.get_config_object().close(con.m_connection_id); - LOG_DEBUG_CC(con, "CONNECTION HANDSHAKED OK AND CLOSED."); + zone.m_net_server.get_config_object().close(con->m_connection_id); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -977,7 +1099,9 @@ namespace nodetool time_t last_seen; time(&last_seen); pe_local.last_seen = static_cast<int64_t>(last_seen); - m_peerlist.append_with_peer_white(pe_local); + pe_local.pruning_seed = con->m_pruning_seed; + pe_local.rpc_port = con->m_rpc_port; + zone.m_peerlist.append_with_peer_white(pe_local); //update last seen and push it to peerlist manager anchor_peerlist_entry ape = AUTO_VAL_INIT(ape); @@ -985,51 +1109,46 @@ namespace nodetool ape.id = pi; ape.first_seen = first_seen_stamp ? first_seen_stamp : time(nullptr); - m_peerlist.append_with_peer_anchor(ape); + zone.m_peerlist.append_with_peer_anchor(ape); - LOG_DEBUG_CC(con, "CONNECTION HANDSHAKED OK."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK."); return true; } template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::check_connection_and_handshake_with_peer(const epee::net_utils::network_address& na, uint64_t last_seen_stamp) { + network_zone& zone = m_network_zones.at(na.get_zone()); + if (zone.m_connect == nullptr) + return false; + LOG_PRINT_L1("Connecting to " << na.str() << "(last_seen: " << (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never") << ")..."); - CHECK_AND_ASSERT_MES(na.get_type_id() == epee::net_utils::ipv4_network_address::ID, false, - "Only IPv4 addresses are supported here"); - const epee::net_utils::ipv4_network_address &ipv4 = na.as<epee::net_utils::ipv4_network_address>(); - - typename net_server::t_connection_context con = AUTO_VAL_INIT(con); - bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), - epee::string_tools::num_to_string_fast(ipv4.port()), - m_config.m_net_config.connection_timeout, - con); - - if (!res) { + auto con = zone.m_connect(zone, na, m_ssl_support); + if (!con) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Connect failed to " << na.str()); + LOG_PRINT_CC_PRIORITY_NODE(is_priority, p2p_connection_context{}, "Connect failed to " << na.str()); return false; } + con->m_anchor = false; peerid_type pi = AUTO_VAL_INIT(pi); - res = do_handshake_with_peer(pi, con, true); - + const bool res = do_handshake_with_peer(pi, *con, true); if (!res) { bool is_priority = is_priority_node(na); - LOG_PRINT_CC_PRIORITY_NODE(is_priority, con, "Failed to HANDSHAKE with peer " << na.str()); + LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str()); return false; } - m_net_server.get_config_object().close(con.m_connection_id); + zone.m_net_server.get_config_object().close(con->m_connection_id); - LOG_DEBUG_CC(con, "CONNECTION HANDSHAKED OK AND CLOSED."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -1055,7 +1174,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist) { for (const auto& pe: anchor_peerlist) { - _note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to anchor peer: " << peerid_type(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1086,49 +1205,80 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::make_new_connection_from_peerlist(bool use_white_list) + bool node_server<t_payload_net_handler>::make_new_connection_from_peerlist(network_zone& zone, bool use_white_list) { - size_t local_peers_count = use_white_list ? m_peerlist.get_white_peers_count():m_peerlist.get_gray_peers_count(); - if(!local_peers_count) - return false;//no peers - - size_t max_random_index = std::min<uint64_t>(local_peers_count -1, 20); + size_t max_random_index = 0; std::set<size_t> tried_peers; size_t try_count = 0; size_t rand_count = 0; - while(rand_count < (max_random_index+1)*3 && try_count < 10 && !m_net_server.is_stop_signal_sent()) + while(rand_count < (max_random_index+1)*3 && try_count < 10 && !zone.m_net_server.is_stop_signal_sent()) { ++rand_count; size_t random_index; + const uint32_t next_needed_pruning_stripe = m_payload_handler.get_next_needed_pruning_stripe().second; - if (use_white_list) { - local_peers_count = m_peerlist.get_white_peers_count(); - if (!local_peers_count) - return false; - max_random_index = std::min<uint64_t>(local_peers_count -1, 20); - random_index = get_random_index_with_fixed_probability(max_random_index); - } else { - local_peers_count = m_peerlist.get_gray_peers_count(); - if (!local_peers_count) + std::deque<size_t> filtered; + const size_t limit = use_white_list ? 20 : std::numeric_limits<size_t>::max(); + size_t idx = 0; + zone.m_peerlist.foreach (use_white_list, [&filtered, &idx, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ + if (filtered.size() >= limit) return false; - random_index = crypto::rand<size_t>() % local_peers_count; + if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) + filtered.push_back(idx); + else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) + filtered.push_front(idx); + ++idx; + return true; + }); + if (filtered.empty()) + { + MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe); + return false; + } + if (use_white_list) + { + // if using the white list, we first pick in the set of peers we've already been using earlier + random_index = get_random_index_with_fixed_probability(std::min<uint64_t>(filtered.size() - 1, 20)); + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + if (next_needed_pruning_stripe > 0 && next_needed_pruning_stripe <= (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES) && !m_used_stripe_peers[next_needed_pruning_stripe-1].empty()) + { + const epee::net_utils::network_address na = m_used_stripe_peers[next_needed_pruning_stripe-1].front(); + m_used_stripe_peers[next_needed_pruning_stripe-1].pop_front(); + for (size_t i = 0; i < filtered.size(); ++i) + { + peerlist_entry pe; + if (zone.m_peerlist.get_white_peer_by_index(pe, filtered[i]) && pe.adr == na) + { + MDEBUG("Reusing stripe " << next_needed_pruning_stripe << " peer " << pe.adr.str()); + random_index = i; + break; + } + } + } } + else + random_index = crypto::rand<size_t>() % filtered.size(); - CHECK_AND_ASSERT_MES(random_index < local_peers_count, false, "random_starter_index < peers_local.size() failed!!"); + CHECK_AND_ASSERT_MES(random_index < filtered.size(), false, "random_index < filtered.size() failed!!"); + random_index = filtered[random_index]; + CHECK_AND_ASSERT_MES(random_index < (use_white_list ? zone.m_peerlist.get_white_peers_count() : zone.m_peerlist.get_gray_peers_count()), + false, "random_index < peers size failed!!"); if(tried_peers.count(random_index)) continue; tried_peers.insert(random_index); peerlist_entry pe = AUTO_VAL_INIT(pe); - bool r = use_white_list ? m_peerlist.get_white_peer_by_index(pe, random_index):m_peerlist.get_gray_peer_by_index(pe, random_index); + bool r = use_white_list ? zone.m_peerlist.get_white_peer_by_index(pe, random_index):zone.m_peerlist.get_gray_peer_by_index(pe, random_index); CHECK_AND_ASSERT_MES(r, false, "Failed to get random peer from peerlist(white:" << use_white_list << ")"); ++try_count; - _note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to " << (use_white_list ? "white" : "gray") << " list peer: " << + peerid_to_string(pe.id) << " " << pe.adr.str() << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << + " (stripe " << next_needed_pruning_stripe << " needed)"); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1142,6 +1292,7 @@ namespace nodetool continue; MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() + << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << " " << "[peer_list=" << (use_white_list ? white : gray) << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); @@ -1163,9 +1314,10 @@ namespace nodetool size_t try_count = 0; size_t current_index = crypto::rand<size_t>()%m_seed_nodes.size(); + const net_server& server = m_network_zones.at(epee::net_utils::zone::public_).m_net_server; while(true) { - if(m_net_server.is_stop_signal_sent()) + if(server.is_stop_signal_sent()) return false; if(try_to_connect_and_handshake_with_new_peer(m_seed_nodes[current_index], true)) @@ -1204,13 +1356,17 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::connections_maker() { + using zone_type = epee::net_utils::zone; + if (m_offline) return true; if (!connect_to_peerlist(m_exclusive_peers)) return false; if (!m_exclusive_peers.empty()) return true; - size_t start_conn_count = get_outgoing_connections_count(); - if(!m_peerlist.get_white_peers_count() && m_seed_nodes.size()) + // Only have seeds in the public zone right now. + + size_t start_conn_count = get_public_outgoing_connections_count(); + if(!get_public_white_peers_count() && m_seed_nodes.size()) { if (!connect_to_seed()) return false; @@ -1218,34 +1374,49 @@ namespace nodetool if (!connect_to_peerlist(m_priority_peers)) return false; - size_t expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; - - size_t conn_count = get_outgoing_connections_count(); - if(conn_count < m_config.m_net_config.max_out_connection_count) + for(auto& zone : m_network_zones) { - if(conn_count < expected_white_connections) - { - //start from anchor list - if(!make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)) - return false; - //then do white list - if(!make_expected_connections_count(white, expected_white_connections)) - return false; - //then do grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; - }else + size_t base_expected_white_connections = (zone.second.m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; + + size_t conn_count = get_outgoing_connections_count(zone.second); + while(conn_count < zone.second.m_config.m_net_config.max_out_connection_count) { - //start from grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; - //and then do white list - if(!make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count)) + const size_t expected_white_connections = m_payload_handler.get_next_needed_pruning_stripe().second ? zone.second.m_config.m_net_config.max_out_connection_count : base_expected_white_connections; + if(conn_count < expected_white_connections) + { + //start from anchor list + while (get_outgoing_connections_count(zone.second) < P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT + && make_expected_connections_count(zone.second, anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)); + //then do white list + while (get_outgoing_connections_count(zone.second) < expected_white_connections + && make_expected_connections_count(zone.second, white, expected_white_connections)); + //then do grey list + while (get_outgoing_connections_count(zone.second) < zone.second.m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(zone.second, gray, zone.second.m_config.m_net_config.max_out_connection_count)); + }else + { + //start from grey list + while (get_outgoing_connections_count(zone.second) < zone.second.m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(zone.second, gray, zone.second.m_config.m_net_config.max_out_connection_count)); + //and then do white list + while (get_outgoing_connections_count(zone.second) < zone.second.m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(zone.second, white, zone.second.m_config.m_net_config.max_out_connection_count)); + } + if(zone.second.m_net_server.is_stop_signal_sent()) return false; + size_t new_conn_count = get_outgoing_connections_count(zone.second); + if (new_conn_count <= conn_count) + { + // we did not make any connection, sleep a bit to avoid a busy loop in case we don't have + // any peers to try, then break so we will try seeds to get more peers + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + break; + } + conn_count = new_conn_count; } } - if (start_conn_count == get_outgoing_connections_count() && start_conn_count < m_config.m_net_config.max_out_connection_count) + if (start_conn_count == get_public_outgoing_connections_count() && start_conn_count < m_network_zones.at(zone_type::public_).m_config.m_net_config.max_out_connection_count) { MINFO("Failed to connect to any, trying seeds"); if (!connect_to_seed()) @@ -1256,71 +1427,128 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::make_expected_connections_count(PeerType peer_type, size_t expected_connections) + bool node_server<t_payload_net_handler>::make_expected_connections_count(network_zone& zone, PeerType peer_type, size_t expected_connections) { if (m_offline) - return true; + return false; std::vector<anchor_peerlist_entry> apl; if (peer_type == anchor) { - m_peerlist.get_and_empty_anchor_peerlist(apl); + zone.m_peerlist.get_and_empty_anchor_peerlist(apl); } - size_t conn_count = get_outgoing_connections_count(); + size_t conn_count = get_outgoing_connections_count(zone); //add new connections from white peers - while(conn_count < expected_connections) + if(conn_count < expected_connections) { - if(m_net_server.is_stop_signal_sent()) + if(zone.m_net_server.is_stop_signal_sent()) return false; + MDEBUG("Making expected connection, type " << peer_type << ", " << conn_count << "/" << expected_connections << " connections"); + if (peer_type == anchor && !make_new_connection_from_anchor_peerlist(apl)) { - break; + return false; } - if (peer_type == white && !make_new_connection_from_peerlist(true)) { - break; + if (peer_type == white && !make_new_connection_from_peerlist(zone, true)) { + return false; } - if (peer_type == gray && !make_new_connection_from_peerlist(false)) { - break; + if (peer_type == gray && !make_new_connection_from_peerlist(zone, false)) { + return false; } - - conn_count = get_outgoing_connections_count(); } return true; } - //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - size_t node_server<t_payload_net_handler>::get_outgoing_connections_count() + size_t node_server<t_payload_net_handler>::get_public_outgoing_connections_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return get_outgoing_connections_count(public_zone->second); + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + size_t node_server<t_payload_net_handler>::get_incoming_connections_count(network_zone& zone) { size_t count = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + zone.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - if(!cntxt.m_is_income) + if(cntxt.m_is_income) ++count; return true; }); - return count; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - size_t node_server<t_payload_net_handler>::get_incoming_connections_count() + size_t node_server<t_payload_net_handler>::get_outgoing_connections_count(network_zone& zone) { size_t count = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + zone.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { - if(cntxt.m_is_income) + if(!cntxt.m_is_income) ++count; return true; }); - return count; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> + size_t node_server<t_payload_net_handler>::get_outgoing_connections_count() + { + size_t count = 0; + for(auto& zone : m_network_zones) + count += get_outgoing_connections_count(zone.second); + return count; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + size_t node_server<t_payload_net_handler>::get_incoming_connections_count() + { + size_t count = 0; + for (auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if(cntxt.m_is_income) + ++count; + return true; + }); + } + return count; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + size_t node_server<t_payload_net_handler>::get_public_white_peers_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return public_zone->second.m_peerlist.get_white_peers_count(); + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + size_t node_server<t_payload_net_handler>::get_public_gray_peers_count() + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone == m_network_zones.end()) + return 0; + return public_zone->second.m_peerlist.get_gray_peers_count(); + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::get_public_peerlist(std::vector<peerlist_entry>& gray, std::vector<peerlist_entry>& white) + { + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end()) + public_zone->second.m_peerlist.get_peerlist(gray, white); + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::idle_worker() { m_peer_handshake_idle_maker_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::peer_sync_idle_maker, this)); @@ -1336,9 +1564,11 @@ namespace nodetool { if (m_offline) return true; - if (get_incoming_connections_count() == 0) + + const auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end() && get_incoming_connections_count(public_zone->second) == 0) { - if (m_hide_my_port || m_config.m_net_config.max_in_connection_count == 0) + if (m_hide_my_port || public_zone->second.m_config.m_net_config.max_in_connection_count == 0) { MGINFO("Incoming connections disabled, enable them for full connectivity"); } @@ -1357,15 +1587,18 @@ namespace nodetool MDEBUG("STARTED PEERLIST IDLE HANDSHAKE"); typedef std::list<std::pair<epee::net_utils::connection_context_base, peerid_type> > local_connects_type; local_connects_type cncts; - m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntxt) + for(auto& zone : m_network_zones) { - if(cntxt.peer_id && !cntxt.m_in_timedsync) + zone.second.m_net_server.get_config_object().foreach_connection([&](p2p_connection_context& cntxt) { - cntxt.m_in_timedsync = true; - cncts.push_back(local_connects_type::value_type(cntxt, cntxt.peer_id));//do idle sync only with handshaked connections - } - return true; - }); + if(cntxt.peer_id && !cntxt.m_in_timedsync) + { + cntxt.m_in_timedsync = true; + cncts.push_back(local_connects_type::value_type(cntxt, cntxt.peer_id));//do idle sync only with handshaked connections + } + return true; + }); + } std::for_each(cncts.begin(), cncts.end(), [&](const typename local_connects_type::value_type& vl){do_peer_timed_sync(vl.first, vl.second);}); @@ -1389,6 +1622,9 @@ namespace nodetool return false; } be.last_seen += delta; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); +#endif } return true; } @@ -1400,29 +1636,41 @@ namespace nodetool std::vector<peerlist_entry> peerlist_ = peerlist; if(!fix_time_delta(peerlist_, local_time, delta)) return false; + + const epee::net_utils::zone zone = context.m_remote_address.get_zone(); + for(const auto& peer : peerlist_) + { + if(peer.adr.get_zone() != zone) + { + MWARNING(context << " sent peerlist from another zone, dropping"); + return false; + } + } + LOG_DEBUG_CC(context, "REMOTE PEERLIST: TIME_DELTA: " << delta << ", remote peerlist size=" << peerlist_.size()); LOG_DEBUG_CC(context, "REMOTE PEERLIST: " << print_peerlist_to_string(peerlist_)); - return m_peerlist.merge_peerlist(peerlist_); + return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::get_local_node_data(basic_node_data& node_data) + bool node_server<t_payload_net_handler>::get_local_node_data(basic_node_data& node_data, const network_zone& zone) { time_t local_time; time(&local_time); - node_data.local_time = local_time; - node_data.peer_id = m_config.m_peer_id; - if(!m_hide_my_port) + node_data.local_time = local_time; // \TODO This can be an identifying value across zones (public internet to tor/i2p) ... + node_data.peer_id = zone.m_config.m_peer_id; + if(!m_hide_my_port && zone.m_can_pingback) node_data.my_port = m_external_port ? m_external_port : m_listening_port; else node_data.my_port = 0; + node_data.rpc_port = zone.m_can_pingback ? m_rpc_port : 0; node_data.network_id = m_network_id; return true; } //----------------------------------------------------------------------------------- #ifdef ALLOW_DEBUG_COMMANDS template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::check_trust(const proof_of_trust& tr) + bool node_server<t_payload_net_handler>::check_trust(const proof_of_trust& tr, const epee::net_utils::zone zone_type) { uint64_t local_time = time(NULL); uint64_t time_delata = local_time > tr.time ? local_time - tr.time: tr.time - local_time; @@ -1436,9 +1684,11 @@ namespace nodetool MWARNING("check_trust failed to check time conditions, last_stat_request_time=" << m_last_stat_request_time << ", proof_time=" << tr.time); return false; } - if(m_config.m_peer_id != tr.peer_id) + + const network_zone& zone = m_network_zones.at(zone_type); + if(zone.m_config.m_peer_id != tr.peer_id) { - MWARNING("check_trust failed: peer_id mismatch (passed " << tr.peer_id << ", expected " << m_config.m_peer_id<< ")"); + MWARNING("check_trust failed: peer_id mismatch (passed " << tr.peer_id << ", expected " << zone.m_config.m_peer_id<< ")"); return false; } crypto::public_key pk = AUTO_VAL_INIT(pk); @@ -1457,12 +1707,12 @@ namespace nodetool template<class t_payload_net_handler> int node_server<t_payload_net_handler>::handle_get_stat_info(int command, typename COMMAND_REQUEST_STAT_INFO::request& arg, typename COMMAND_REQUEST_STAT_INFO::response& rsp, p2p_connection_context& context) { - if(!check_trust(arg.tr)) + if(!check_trust(arg.tr, context.m_remote_address.get_zone())) { drop_connection(context); return 1; } - rsp.connections_count = m_net_server.get_config_object().get_connections_count(); + rsp.connections_count = get_connections_count(); rsp.incoming_connections_count = rsp.connections_count - get_outgoing_connections_count(); rsp.version = MONERO_VERSION_FULL; rsp.os_version = tools::get_os_version_string(); @@ -1473,12 +1723,12 @@ namespace nodetool template<class t_payload_net_handler> int node_server<t_payload_net_handler>::handle_get_network_state(int command, COMMAND_REQUEST_NETWORK_STATE::request& arg, COMMAND_REQUEST_NETWORK_STATE::response& rsp, p2p_connection_context& context) { - if(!check_trust(arg.tr)) + if(!check_trust(arg.tr, context.m_remote_address.get_zone())) { drop_connection(context); return 1; } - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { connection_entry ce; ce.adr = cntxt.m_remote_address; @@ -1488,8 +1738,9 @@ namespace nodetool return true; }); - m_peerlist.get_peerlist_full(rsp.local_peerlist_gray, rsp.local_peerlist_white); - rsp.my_id = m_config.m_peer_id; + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + zone.m_peerlist.get_peerlist(rsp.local_peerlist_gray, rsp.local_peerlist_white); + rsp.my_id = zone.m_config.m_peer_id; rsp.local_time = time(NULL); return 1; } @@ -1497,7 +1748,7 @@ namespace nodetool template<class t_payload_net_handler> int node_server<t_payload_net_handler>::handle_get_peer_id(int command, COMMAND_REQUEST_PEER_ID::request& arg, COMMAND_REQUEST_PEER_ID::response& rsp, p2p_connection_context& context) { - rsp.my_id = m_config.m_peer_id; + rsp.my_id = m_network_zones.at(context.m_remote_address.get_zone()).m_config.m_peer_id; return 1; } #endif @@ -1505,63 +1756,73 @@ namespace nodetool template<class t_payload_net_handler> int node_server<t_payload_net_handler>::handle_get_support_flags(int command, COMMAND_REQUEST_SUPPORT_FLAGS::request& arg, COMMAND_REQUEST_SUPPORT_FLAGS::response& rsp, p2p_connection_context& context) { - rsp.support_flags = m_config.m_support_flags; + rsp.support_flags = m_network_zones.at(context.m_remote_address.get_zone()).m_config.m_support_flags; return 1; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> void node_server<t_payload_net_handler>::request_callback(const epee::net_utils::connection_context_base& context) { - m_net_server.get_config_object().request_callback(context.m_connection_id); + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().request_callback(context.m_connection_id); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid> &connections) + bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { + std::sort(connections.begin(), connections.end()); + auto zone = m_network_zones.begin(); for(const auto& c_id: connections) { - m_net_server.get_config_object().notify(command, data_buff, c_id); + for (;;) + { + if (zone == m_network_zones.end()) + { + MWARNING("Unable to relay all messages, " << epee::net_utils::zone_to_string(c_id.first) << " not available"); + return false; + } + if (c_id.first <= zone->first) + break; + + ++zone; + } + if (zone->first == c_id.first) + zone->second.m_net_server.get_config_object().notify(command, data_buff, c_id.second); } return true; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) - { - std::list<boost::uuids::uuid> connections; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) - { - if(cntxt.peer_id && context.m_connection_id != cntxt.m_connection_id) - connections.push_back(cntxt.m_connection_id); - return true; - }); - return relay_notify_to_list(command, data_buff, connections); - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> void node_server<t_payload_net_handler>::callback(p2p_connection_context& context) { m_payload_handler.on_callback(context); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) { - int res = m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); + if(is_filtered_command(context.m_remote_address, command)) + return false; + + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + int res = zone.m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { - int res = m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); + if(is_filtered_command(context.m_remote_address, command)) + return false; + + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + int res = zone.m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::drop_connection(const epee::net_utils::connection_context_base& context) { - m_net_server.get_config_object().close(context.m_connection_id); + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id); return true; } //----------------------------------------------------------------------------------- @@ -1571,18 +1832,21 @@ namespace nodetool if(!node_data.my_port) return false; - CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::ID, false, + CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), false, "Only IPv4 addresses are supported here"); const epee::net_utils::network_address na = context.m_remote_address; uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip(); - if(!m_peerlist.is_host_allowed(context.m_remote_address)) + network_zone& zone = m_network_zones.at(na.get_zone()); + + if(!zone.m_peerlist.is_host_allowed(context.m_remote_address)) return false; + std::string ip = epee::string_tools::get_ip_string_from_int32(actual_ip); std::string port = epee::string_tools::num_to_string_fast(node_data.my_port); epee::net_utils::network_address address{epee::net_utils::ipv4_network_address(actual_ip, node_data.my_port)}; peerid_type pr = node_data.peer_id; - bool r = m_net_server.connect_async(ip, port, m_config.m_net_config.ping_connection_timeout, [cb, /*context,*/ address, pr, this]( + bool r = zone.m_net_server.connect_async(ip, port, zone.m_config.m_net_config.ping_connection_timeout, [cb, /*context,*/ address, pr, this]( const typename net_server::t_connection_context& ping_context, const boost::system::error_code& ec)->bool { @@ -1602,7 +1866,9 @@ namespace nodetool // GCC 5.1.0 gives error with second use of uint64_t (peerid_type) variable. peerid_type pr_ = pr; - bool inv_call_res = epee::net_utils::async_invoke_remote_command2<COMMAND_PING::response>(ping_context.m_connection_id, COMMAND_PING::ID, req, m_net_server.get_config_object(), + network_zone& zone = m_network_zones.at(address.get_zone()); + + bool inv_call_res = epee::net_utils::async_invoke_remote_command2<COMMAND_PING::response>(ping_context.m_connection_id, COMMAND_PING::ID, req, zone.m_net_server.get_config_object(), [=](int code, const COMMAND_PING::response& rsp, p2p_connection_context& context) { if(code <= 0) @@ -1611,20 +1877,21 @@ namespace nodetool return; } + network_zone& zone = m_network_zones.at(address.get_zone()); if(rsp.status != PING_OK_RESPONSE_STATUS_TEXT || pr != rsp.peer_id) { LOG_WARNING_CC(ping_context, "back ping invoke wrong response \"" << rsp.status << "\" from" << address.str() << ", hsh_peer_id=" << pr_ << ", rsp.peer_id=" << rsp.peer_id); - m_net_server.get_config_object().close(ping_context.m_connection_id); + zone.m_net_server.get_config_object().close(ping_context.m_connection_id); return; } - m_net_server.get_config_object().close(ping_context.m_connection_id); + zone.m_net_server.get_config_object().close(ping_context.m_connection_id); cb(); }); if(!inv_call_res) { LOG_WARNING_CC(ping_context, "back ping invoke failed to " << address.str()); - m_net_server.get_config_object().close(ping_context.m_connection_id); + zone.m_net_server.get_config_object().close(ping_context.m_connection_id); return false; } return true; @@ -1639,13 +1906,16 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::try_get_support_flags(const p2p_connection_context& context, std::function<void(p2p_connection_context&, const uint32_t&)> f) { + if(context.m_remote_address.get_zone() != epee::net_utils::zone::public_) + return false; + COMMAND_REQUEST_SUPPORT_FLAGS::request support_flags_request; bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_REQUEST_SUPPORT_FLAGS::response> ( context.m_connection_id, COMMAND_REQUEST_SUPPORT_FLAGS::ID, support_flags_request, - m_net_server.get_config_object(), + m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object(), [=](int code, const typename COMMAND_REQUEST_SUPPORT_FLAGS::response& rsp, p2p_connection_context& context_) { if(code < 0) @@ -1674,8 +1944,24 @@ namespace nodetool //fill response rsp.local_time = time(NULL); - m_peerlist.get_peerlist_head(rsp.local_peerlist_new); + + const epee::net_utils::zone zone_type = context.m_remote_address.get_zone(); + network_zone& zone = m_network_zones.at(zone_type); + + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new); m_payload_handler.get_payload_sync_data(rsp.payload_data); + + /* Tor/I2P nodes receiving connections via forwarding (from tor/i2p daemon) + do not know the address of the connecting peer. This is relayed to them, + iff the node has setup an inbound hidden service. The other peer will have + to use the random peer_id value to link the two. My initial thought is that + the inbound peer should leave the other side marked as `<unknown tor host>`, + etc., because someone could give faulty addresses over Tor/I2P to get the + real peer with that identity banned/blacklisted. */ + + if(!context.m_is_income && zone.m_our_address.get_zone() == zone_type) + rsp.local_peerlist_new.push_back(peerlist_entry{zone.m_our_address, zone.m_config.m_peer_id, std::time(nullptr)}); + LOG_DEBUG_CC(context, "COMMAND_TIMED_SYNC"); return 1; } @@ -1686,7 +1972,7 @@ namespace nodetool if(arg.node_data.network_id != m_network_id) { - LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << epee::string_tools::get_str_from_guid_a(arg.node_data.network_id)); + LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id); drop_connection(context); add_host_fail(context.m_remote_address); return 1; @@ -1707,7 +1993,9 @@ namespace nodetool return 1; } - if (m_current_number_of_in_peers >= m_config.m_net_config.max_in_connection_count) // in peers limit + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + + if (zone.m_current_number_of_in_peers >= zone.m_config.m_net_config.max_in_connection_count) // in peers limit { LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came, but already have max incoming connections, so dropping this one."); drop_connection(context); @@ -1731,15 +2019,16 @@ namespace nodetool //associate peer_id with this connection context.peer_id = arg.node_data.peer_id; context.m_in_timedsync = false; + context.m_rpc_port = arg.node_data.rpc_port; - if(arg.node_data.peer_id != m_config.m_peer_id && arg.node_data.my_port) + if(arg.node_data.peer_id != zone.m_config.m_peer_id && arg.node_data.my_port && zone.m_can_pingback) { peerid_type peer_id_l = arg.node_data.peer_id; uint32_t port_l = arg.node_data.my_port; //try ping to be sure that we can add this peer to peer_list try_ping(arg.node_data, context, [peer_id_l, port_l, context, this]() { - CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::ID, void(), + CHECK_AND_ASSERT_MES(context.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), void(), "Only IPv4 addresses are supported here"); //called only(!) if success pinged, update local peerlist peerlist_entry pe; @@ -1749,7 +2038,9 @@ namespace nodetool time(&last_seen); pe.last_seen = static_cast<int64_t>(last_seen); pe.id = peer_id_l; - this->m_peerlist.append_with_peer_white(pe); + pe.pruning_seed = context.m_pruning_seed; + pe.rpc_port = context.m_rpc_port; + this->m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.append_with_peer_white(pe); LOG_DEBUG_CC(context, "PING SUCCESS " << context.m_remote_address.host_str() << ":" << port_l); }); } @@ -1760,8 +2051,8 @@ namespace nodetool }); //fill response - m_peerlist.get_peerlist_head(rsp.local_peerlist_new); - get_local_node_data(rsp.node_data); + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new); + get_local_node_data(rsp.node_data, zone); m_payload_handler.get_payload_sync_data(rsp.payload_data); LOG_DEBUG_CC(context, "COMMAND_HANDSHAKE"); return 1; @@ -1772,7 +2063,7 @@ namespace nodetool { LOG_DEBUG_CC(context, "COMMAND_PING"); rsp.status = PING_OK_RESPONSE_STATUS_TEXT; - rsp.peer_id = m_config.m_peer_id; + rsp.peer_id = m_network_zones.at(context.m_remote_address.get_zone()).m_config.m_peer_id; return 1; } //----------------------------------------------------------------------------------- @@ -1781,7 +2072,8 @@ namespace nodetool { std::vector<peerlist_entry> pl_white; std::vector<peerlist_entry> pl_gray; - m_peerlist.get_peerlist_full(pl_gray, pl_white); + for (auto& zone : m_network_zones) + zone.second.m_peerlist.get_peerlist(pl_gray, pl_white); MINFO(ENDL << "Peerlist white:" << ENDL << print_peerlist_to_string(pl_white) << ENDL << "Peerlist gray:" << ENDL << print_peerlist_to_string(pl_gray) ); return true; } @@ -1798,14 +2090,17 @@ namespace nodetool { std::stringstream ss; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + for (auto& zone : m_network_zones) { - ss << cntxt.m_remote_address.str() - << " \t\tpeer_id " << cntxt.peer_id - << " \t\tconn_id " << epee::string_tools::get_str_from_guid_a(cntxt.m_connection_id) << (cntxt.m_is_income ? " INC":" OUT") - << std::endl; - return true; - }); + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + ss << cntxt.m_remote_address.str() + << " \t\tpeer_id " << cntxt.peer_id + << " \t\tconn_id " << cntxt.m_connection_id << (cntxt.m_is_income ? " INC":" OUT") + << std::endl; + return true; + }); + } std::string s = ss.str(); return s; } @@ -1819,11 +2114,12 @@ namespace nodetool template<class t_payload_net_handler> void node_server<t_payload_net_handler>::on_connection_close(p2p_connection_context& context) { - if (!m_net_server.is_stop_signal_sent() && !context.m_is_income) { + network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + if (!zone.m_net_server.is_stop_signal_sent() && !context.m_is_income) { epee::net_utils::network_address na = AUTO_VAL_INIT(na); na = context.m_remote_address; - m_peerlist.remove_from_peer_anchor(na); + zone.m_peerlist.remove_from_peer_anchor(na); } m_payload_handler.on_connection_close(context); @@ -1840,9 +2136,10 @@ namespace nodetool template<class t_payload_net_handler> template <class Container> bool node_server<t_payload_net_handler>::connect_to_peerlist(const Container& peers) { + const network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_); for(const epee::net_utils::network_address& na: peers) { - if(m_net_server.is_stop_signal_sent()) + if(public_zone.m_net_server.is_stop_signal_sent()) return false; if(is_addr_connected(na)) @@ -1861,16 +2158,16 @@ namespace nodetool for(const std::string& pr_str: perrs) { - epee::net_utils::network_address na = AUTO_VAL_INIT(na); const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; - bool r = parse_peer_from_string(na, pr_str, default_port); - if (r) + expect<epee::net_utils::network_address> adr = net::get_network_address(pr_str, default_port); + if (adr) { - container.push_back(na); + add_zone(adr->get_zone()); + container.push_back(std::move(*adr)); continue; } std::vector<epee::net_utils::network_address> resolved_addrs; - r = append_net_address(resolved_addrs, pr_str, default_port); + bool r = append_net_address(resolved_addrs, pr_str, default_port); CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); for (const epee::net_utils::network_address& addr : resolved_addrs) { @@ -1882,37 +2179,47 @@ namespace nodetool } template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) + bool node_server<t_payload_net_handler>::set_max_out_peers(network_zone& zone, int64_t max) { if(max == -1) { - m_config.m_net_config.max_out_connection_count = P2P_DEFAULT_CONNECTIONS_COUNT; + zone.m_config.m_net_config.max_out_connection_count = P2P_DEFAULT_CONNECTIONS_COUNT; return true; } - m_config.m_net_config.max_out_connection_count = max; + zone.m_config.m_net_config.max_out_connection_count = max; return true; } template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max) + bool node_server<t_payload_net_handler>::set_max_in_peers(network_zone& zone, int64_t max) { - if(max == -1) { - m_config.m_net_config.max_in_connection_count = -1; - return true; - } - m_config.m_net_config.max_in_connection_count = max; + zone.m_config.m_net_config.max_in_connection_count = max; return true; } template<class t_payload_net_handler> - void node_server<t_payload_net_handler>::delete_out_connections(size_t count) + void node_server<t_payload_net_handler>::change_max_out_public_peers(size_t count) { - m_net_server.get_config_object().del_out_connections(count); + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end()) + { + const auto current = public_zone->second.m_config.m_net_config.max_out_connection_count; + public_zone->second.m_config.m_net_config.max_out_connection_count = count; + if(current > count) + public_zone->second.m_net_server.get_config_object().del_out_connections(current - count); + } } template<class t_payload_net_handler> - void node_server<t_payload_net_handler>::delete_in_connections(size_t count) + void node_server<t_payload_net_handler>::change_max_in_public_peers(size_t count) { - m_net_server.get_config_object().del_in_connections(count); + auto public_zone = m_network_zones.find(epee::net_utils::zone::public_); + if (public_zone != m_network_zones.end()) + { + const auto current = public_zone->second.m_config.m_net_config.max_in_connection_count; + public_zone->second.m_config.m_net_config.max_in_connection_count = count; + if(current > count) + public_zone->second.m_net_server.get_config_object().del_in_connections(current - count); + } } template<class t_payload_net_handler> @@ -1985,10 +2292,13 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::has_too_many_connections(const epee::net_utils::network_address &address) { + if (address.get_zone() != epee::net_utils::zone::public_) + return false; // Unable to determine how many connections from host + const size_t max_connections = 1; size_t count = 0; - m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { if (cntxt.m_is_income && cntxt.m_remote_address.is_same_host(address)) { count++; @@ -2009,31 +2319,68 @@ namespace nodetool { if (m_offline) return true; if (!m_exclusive_peers.empty()) return true; + if (m_payload_handler.needs_new_sync_connections()) return true; - peerlist_entry pe = AUTO_VAL_INIT(pe); - - if (m_net_server.is_stop_signal_sent()) - return false; - - if (!m_peerlist.get_random_gray_peer(pe)) { + for (auto& zone : m_network_zones) + { + if (zone.second.m_net_server.is_stop_signal_sent()) return false; - } - - bool success = check_connection_and_handshake_with_peer(pe.adr, pe.last_seen); - if (!success) { - m_peerlist.remove_from_peer_gray(pe); + if (zone.second.m_connect == nullptr) + continue; - LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + peerlist_entry pe{}; + if (!zone.second.m_peerlist.get_random_gray_peer(pe)) + continue; - return true; + if (!check_connection_and_handshake_with_peer(pe.adr, pe.last_seen)) + { + zone.second.m_peerlist.remove_from_peer_gray(pe); + LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + } + else + { + zone.second.m_peerlist.set_peer_just_seen(pe.id, pe.adr, pe.pruning_seed, pe.rpc_port); + LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + } } + return true; + } - m_peerlist.set_peer_just_seen(pe.id, pe.adr); + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("adding stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + m_used_stripe_peers[index].push_back(context.m_remote_address); + } - LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("removing stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + } - return true; + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::clear_used_stripe_peers() + { + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("clearing used stripe peers"); + for (auto &e: m_used_stripe_peers) + e.clear(); } template<class t_payload_net_handler> @@ -2124,4 +2471,37 @@ namespace nodetool MINFO("No IGD was found."); } } + + template<typename t_payload_net_handler> + boost::optional<p2p_connection_context_t<typename t_payload_net_handler::connection_context>> + node_server<t_payload_net_handler>::socks_connect(network_zone& zone, const epee::net_utils::network_address& remote, epee::net_utils::ssl_support_t ssl_support) + { + auto result = socks_connect_internal(zone.m_net_server.get_stop_signal(), zone.m_net_server.get_io_service(), zone.m_proxy_address, remote); + if (result) // if no error + { + p2p_connection_context context{}; + if (zone.m_net_server.add_connection(context, std::move(*result), remote, ssl_support)) + return {std::move(context)}; + } + return boost::none; + } + + template<typename t_payload_net_handler> + boost::optional<p2p_connection_context_t<typename t_payload_net_handler::connection_context>> + node_server<t_payload_net_handler>::public_connect(network_zone& zone, epee::net_utils::network_address const& na, epee::net_utils::ssl_support_t ssl_support) + { + CHECK_AND_ASSERT_MES(na.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id(), boost::none, + "Only IPv4 addresses are supported here"); + const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); + + typename net_server::t_connection_context con{}; + const bool res = zone.m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), + epee::string_tools::num_to_string_fast(ipv4.port()), + zone.m_config.m_net_config.connection_timeout, + con, "0.0.0.0", ssl_support); + + if (res) + return {std::move(con)}; + return boost::none; + } } diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 218250efa..26451b333 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,8 @@ #pragma once #include <boost/uuid/uuid.hpp> +#include <utility> +#include <vector> #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" @@ -43,37 +45,36 @@ namespace nodetool template<class t_connection_context> struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid>& connections)=0; - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; - virtual uint64_t get_connections_count()=0; + virtual uint64_t get_public_connections_count()=0; + virtual size_t get_zone_count() const=0; virtual void for_each_connection(std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0; virtual bool for_connection(const boost::uuids::uuid&, std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0; virtual bool block_host(const epee::net_utils::network_address &address, time_t seconds = 0)=0; virtual bool unblock_host(const epee::net_utils::network_address &address)=0; virtual std::map<std::string, time_t> get_blocked_hosts()=0; virtual bool add_host_fail(const epee::net_utils::network_address &address)=0; + virtual void add_used_stripe_peer(const t_connection_context &context)=0; + virtual void remove_used_stripe_peer(const t_connection_context &context)=0; + virtual void clear_used_stripe_peers()=0; }; template<class t_connection_context> struct p2p_endpoint_stub: public i_p2p_endpoint<t_connection_context> { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid>& connections) + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { return false; } - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - return false; - } - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) { return true; } @@ -94,7 +95,12 @@ namespace nodetool return false; } - virtual uint64_t get_connections_count() + virtual size_t get_zone_count() const + { + return 0; + } + + virtual uint64_t get_public_connections_count() { return false; } @@ -114,5 +120,14 @@ namespace nodetool { return true; } + virtual void add_used_stripe_peer(const t_connection_context &context) + { + } + virtual void remove_used_stripe_peer(const t_connection_context &context) + { + } + virtual void clear_used_stripe_peers() + { + } }; } diff --git a/src/p2p/net_peerlist.cpp b/src/p2p/net_peerlist.cpp new file mode 100644 index 000000000..ce5c67fe5 --- /dev/null +++ b/src/p2p/net_peerlist.cpp @@ -0,0 +1,295 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net_peerlist.h" + +#include <algorithm> +#include <functional> +#include <fstream> +#include <iterator> + +#include <boost/archive/binary_iarchive.hpp> +#include <boost/archive/portable_binary_oarchive.hpp> +#include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/filesystem/operations.hpp> +#include <boost/range/join.hpp> +#include <boost/serialization/version.hpp> + +#include "net_peerlist_boost_serialization.h" + + +namespace nodetool +{ + namespace + { + constexpr unsigned CURRENT_PEERLIST_STORAGE_ARCHIVE_VER = 6; + + struct by_zone + { + using zone = epee::net_utils::zone; + + template<typename T> + bool operator()(const T& left, const zone right) const + { + return left.adr.get_zone() < right; + } + + template<typename T> + bool operator()(const zone left, const T& right) const + { + return left < right.adr.get_zone(); + } + + template<typename T, typename U> + bool operator()(const T& left, const U& right) const + { + return left.adr.get_zone() < right.adr.get_zone(); + } + }; + + template<typename Elem, typename Archive> + std::vector<Elem> load_peers(Archive& a, unsigned ver) + { + // at v6, we drop existing peerlists, because annoying change + if (ver < 6) + return {}; + + uint64_t size = 0; + a & size; + + Elem ple{}; + + std::vector<Elem> elems{}; + elems.reserve(size); + while (size--) + { + a & ple; + elems.push_back(std::move(ple)); + } + + return elems; + } + + template<typename Archive, typename Range> + void save_peers(Archive& a, const Range& elems) + { + const uint64_t size = elems.size(); + a & size; + for (const auto& elem : elems) + a & elem; + } + + template<typename T> + std::vector<T> do_take_zone(std::vector<T>& src, epee::net_utils::zone zone) + { + const auto start = std::lower_bound(src.begin(), src.end(), zone, by_zone{}); + const auto end = std::upper_bound(start, src.end(), zone, by_zone{}); + + std::vector<T> out{}; + out.assign(std::make_move_iterator(start), std::make_move_iterator(end)); + src.erase(start, end); + return out; + } + + template<typename Container, typename T> + void add_peers(Container& dest, std::vector<T>&& src) + { + dest.insert(std::make_move_iterator(src.begin()), std::make_move_iterator(src.end())); + } + + template<typename Container, typename Range> + void copy_peers(Container& dest, const Range& src) + { + std::copy(src.begin(), src.end(), std::back_inserter(dest)); + } + } // anonymous + + struct peerlist_join + { + const peerlist_types& ours; + const peerlist_types& other; + }; + + template<typename Archive> + void serialize(Archive& a, peerlist_types& elem, unsigned ver) + { + elem.white = load_peers<peerlist_entry>(a, ver); + elem.gray = load_peers<peerlist_entry>(a, ver); + elem.anchor = load_peers<anchor_peerlist_entry>(a, ver); + + if (ver == 0) + { + // from v1, we do not store the peer id anymore + peerid_type peer_id{}; + a & peer_id; + } + } + + template<typename Archive> + void serialize(Archive& a, peerlist_join elem, unsigned ver) + { + save_peers(a, boost::range::join(elem.ours.white, elem.other.white)); + save_peers(a, boost::range::join(elem.ours.gray, elem.other.gray)); + save_peers(a, boost::range::join(elem.ours.anchor, elem.other.anchor)); + } + + boost::optional<peerlist_storage> peerlist_storage::open(std::istream& src, const bool new_format) + { + try + { + peerlist_storage out{}; + if (new_format) + { + boost::archive::portable_binary_iarchive a{src}; + a >> out.m_types; + } + else + { + boost::archive::binary_iarchive a{src}; + a >> out.m_types; + } + + if (src.good()) + { + std::sort(out.m_types.white.begin(), out.m_types.white.end(), by_zone{}); + std::sort(out.m_types.gray.begin(), out.m_types.gray.end(), by_zone{}); + std::sort(out.m_types.anchor.begin(), out.m_types.anchor.end(), by_zone{}); + return {std::move(out)}; + } + } + catch (const std::exception& e) + {} + + return boost::none; + } + + boost::optional<peerlist_storage> peerlist_storage::open(const std::string& path) + { + std::ifstream src_file{}; + src_file.open( path , std::ios_base::binary | std::ios_base::in); + if(src_file.fail()) + return boost::none; + + boost::optional<peerlist_storage> out = open(src_file, true); + if (!out) + { + // if failed, try reading in unportable mode + boost::filesystem::copy_file(path, path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + src_file.close(); + src_file.open( path , std::ios_base::binary | std::ios_base::in); + if(src_file.fail()) + return boost::none; + + out = open(src_file, false); + if (!out) + { + // This is different from the `return boost::none` cases above. Those + // cases could fail due to bad file permissions, so a shutdown is + // likely more appropriate. + MWARNING("Failed to load p2p config file, falling back to default config"); + out.emplace(); + } + } + + return out; + } + + peerlist_storage::~peerlist_storage() noexcept + {} + + bool peerlist_storage::store(std::ostream& dest, const peerlist_types& other) const + { + try + { + boost::archive::portable_binary_oarchive a{dest}; + const peerlist_join pj{std::cref(m_types), std::cref(other)}; + a << pj; + return dest.good(); + } + catch (const boost::archive::archive_exception& e) + {} + + return false; + } + + bool peerlist_storage::store(const std::string& path, const peerlist_types& other) const + { + std::ofstream dest_file{}; + dest_file.open( path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); + if(dest_file.fail()) + return false; + + return store(dest_file, other); + } + + peerlist_types peerlist_storage::take_zone(epee::net_utils::zone zone) + { + peerlist_types out{}; + out.white = do_take_zone(m_types.white, zone); + out.gray = do_take_zone(m_types.gray, zone); + out.anchor = do_take_zone(m_types.anchor, zone); + return out; + } + + bool peerlist_manager::init(peerlist_types&& peers, bool allow_local_ip) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + + if (!m_peers_white.empty() || !m_peers_gray.empty() || !m_peers_anchor.empty()) + return false; + + add_peers(m_peers_white.get<by_addr>(), std::move(peers.white)); + add_peers(m_peers_gray.get<by_addr>(), std::move(peers.gray)); + add_peers(m_peers_anchor.get<by_addr>(), std::move(peers.anchor)); + m_allow_local_ip = allow_local_ip; + return true; + } + + void peerlist_manager::get_peerlist(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + copy_peers(pl_gray, m_peers_gray.get<by_addr>()); + copy_peers(pl_white, m_peers_white.get<by_addr>()); + } + + void peerlist_manager::get_peerlist(peerlist_types& peers) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + peers.white.reserve(peers.white.size() + m_peers_white.size()); + peers.gray.reserve(peers.gray.size() + m_peers_gray.size()); + peers.anchor.reserve(peers.anchor.size() + m_peers_anchor.size()); + + copy_peers(peers.white, m_peers_white.get<by_addr>()); + copy_peers(peers.gray, m_peers_gray.get<by_addr>()); + copy_peers(peers.anchor, m_peers_anchor.get<by_addr>()); + } +} + +BOOST_CLASS_VERSION(nodetool::peerlist_types, nodetool::CURRENT_PEERLIST_STORAGE_ARCHIVE_VER); +BOOST_CLASS_VERSION(nodetool::peerlist_join, nodetool::CURRENT_PEERLIST_STORAGE_ARCHIVE_VER); + diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index e7aad5abe..ebe0268d8 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,33 +30,67 @@ #pragma once +#include <iosfwd> #include <list> -#include <set> -#include <map> -#include <boost/archive/binary_iarchive.hpp> -#include <boost/archive/portable_binary_oarchive.hpp> -#include <boost/archive/portable_binary_iarchive.hpp> -#include <boost/serialization/version.hpp> +#include <string> +#include <vector> #include <boost/multi_index_container.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/identity.hpp> #include <boost/multi_index/member.hpp> +#include <boost/optional/optional.hpp> #include <boost/range/adaptor/reversed.hpp> -#include "syncobj.h" +#include "cryptonote_config.h" +#include "net/enums.h" #include "net/local_ip.h" #include "p2p_protocol_defs.h" -#include "cryptonote_config.h" -#include "net_peerlist_boost_serialization.h" - - -#define CURRENT_PEERLIST_STORAGE_ARCHIVE_VER 6 +#include "syncobj.h" namespace nodetool { + struct peerlist_types + { + std::vector<peerlist_entry> white; + std::vector<peerlist_entry> gray; + std::vector<anchor_peerlist_entry> anchor; + }; + + class peerlist_storage + { + public: + peerlist_storage() + : m_types{} + {} + //! \return Peers stored in stream `src` in `new_format` (portable archive or older non-portable). + static boost::optional<peerlist_storage> open(std::istream& src, const bool new_format); + + //! \return Peers stored in file at `path` + static boost::optional<peerlist_storage> open(const std::string& path); + + peerlist_storage(peerlist_storage&&) = default; + peerlist_storage(const peerlist_storage&) = delete; + + ~peerlist_storage() noexcept; + + peerlist_storage& operator=(peerlist_storage&&) = default; + peerlist_storage& operator=(const peerlist_storage&) = delete; + + //! Save peers from `this` and `other` in stream `dest`. + bool store(std::ostream& dest, const peerlist_types& other) const; + + //! Save peers from `this` and `other` in one file at `path`. + bool store(const std::string& path, const peerlist_types& other) const; + + //! \return Peers in `zone` and from remove from `this`. + peerlist_types take_zone(epee::net_utils::zone zone); + + private: + peerlist_types m_types; + }; /************************************************************************/ /* */ @@ -64,25 +98,27 @@ namespace nodetool class peerlist_manager { public: - bool init(bool allow_local_ip); - bool deinit(); + bool init(peerlist_types&& peers, bool allow_local_ip); size_t get_white_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_white.size();} size_t get_gray_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_gray.size();} bool merge_peerlist(const std::vector<peerlist_entry>& outer_bs); bool get_peerlist_head(std::vector<peerlist_entry>& bs_head, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); - bool get_peerlist_full(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white); + void get_peerlist(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white); + void get_peerlist(peerlist_types& peers); bool get_white_peer_by_index(peerlist_entry& p, size_t i); bool get_gray_peer_by_index(peerlist_entry& p, size_t i); + template<typename F> bool foreach(bool white, const F &f); bool append_with_peer_white(const peerlist_entry& pr); bool append_with_peer_gray(const peerlist_entry& pr); bool append_with_peer_anchor(const anchor_peerlist_entry& ple); - bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr); + bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed, uint16_t rpc_port); bool set_peer_unreachable(const peerlist_entry& pr); bool is_host_allowed(const epee::net_utils::network_address &address); bool get_random_gray_peer(peerlist_entry& pe); bool remove_from_peer_gray(const peerlist_entry& pe); bool get_and_empty_anchor_peerlist(std::vector<anchor_peerlist_entry>& apl); bool remove_from_peer_anchor(const epee::net_utils::network_address& addr); + bool remove_from_peer_white(const peerlist_entry& pe); private: struct by_time{}; @@ -134,18 +170,6 @@ namespace nodetool > peers_indexed; typedef boost::multi_index_container< - peerlist_entry, - boost::multi_index::indexed_by< - // access by peerlist_entry::id< - boost::multi_index::ordered_unique<boost::multi_index::tag<by_id>, boost::multi_index::member<peerlist_entry,uint64_t,&peerlist_entry::id> >, - // access by peerlist_entry::net_adress - boost::multi_index::ordered_unique<boost::multi_index::tag<by_addr>, boost::multi_index::member<peerlist_entry,epee::net_utils::network_address,&peerlist_entry::adr> >, - // sort by peerlist_entry::last_seen< - boost::multi_index::ordered_non_unique<boost::multi_index::tag<by_time>, boost::multi_index::member<peerlist_entry,int64_t,&peerlist_entry::last_seen> > - > - > peers_indexed_old; - - typedef boost::multi_index_container< anchor_peerlist_entry, boost::multi_index::indexed_by< // access by anchor_peerlist_entry::net_adress @@ -154,56 +178,8 @@ namespace nodetool boost::multi_index::ordered_non_unique<boost::multi_index::tag<by_time>, boost::multi_index::member<anchor_peerlist_entry,int64_t,&anchor_peerlist_entry::first_seen> > > > anchor_peers_indexed; - public: - - template <class Archive, class List, class Element, class t_version_type> - void serialize_peers(Archive &a, List &list, Element ple, const t_version_type ver) - { - if (typename Archive::is_saving()) - { - uint64_t size = list.size(); - a & size; - for (auto p: list) - { - a & p; - } - } - else - { - uint64_t size; - a & size; - list.clear(); - while (size--) - { - a & ple; - list.insert(ple); - } - } - } - - template <class Archive, class t_version_type> - void serialize(Archive &a, const t_version_type ver) - { - // at v6, we drop existing peerlists, because annoying change - if (ver < 6) - return; - - CRITICAL_REGION_LOCAL(m_peerlist_lock); - -#if 0 - // trouble loading more than one peer, can't find why - a & m_peers_white; - a & m_peers_gray; - a & m_peers_anchor; -#else - serialize_peers(a, m_peers_white, peerlist_entry(), ver); - serialize_peers(a, m_peers_gray, peerlist_entry(), ver); - serialize_peers(a, m_peers_anchor, anchor_peerlist_entry(), ver); -#endif - } private: - bool peers_indexed_from_old(const peers_indexed_old& pio, peers_indexed& pi); void trim_white_peerlist(); void trim_gray_peerlist(); @@ -218,34 +194,6 @@ namespace nodetool anchor_peers_indexed m_peers_anchor; }; //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::init(bool allow_local_ip) - { - m_allow_local_ip = allow_local_ip; - return true; - } - //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::deinit() - { - return true; - } - //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::peers_indexed_from_old(const peers_indexed_old& pio, peers_indexed& pi) - { - for(auto x: pio) - { - auto by_addr_it = pi.get<by_addr>().find(x.adr); - if(by_addr_it == pi.get<by_addr>().end()) - { - pi.insert(x); - } - } - - return true; - } - //-------------------------------------------------------------------------------------------------- inline void peerlist_manager::trim_gray_peerlist() { while(m_peers_gray.size() > P2P_LOCAL_GRAY_PEERLIST_LIMIT) @@ -335,29 +283,19 @@ namespace nodetool return true; } //-------------------------------------------------------------------------------------------------- - inline - bool peerlist_manager::get_peerlist_full(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white) - { + template<typename F> inline + bool peerlist_manager::foreach(bool white, const F &f) + { CRITICAL_REGION_LOCAL(m_peerlist_lock); - peers_indexed::index<by_time>::type& by_time_index_gr=m_peers_gray.get<by_time>(); - pl_gray.resize(pl_gray.size() + by_time_index_gr.size()); - for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index_gr)) - { - pl_gray.push_back(vl); - } - - peers_indexed::index<by_time>::type& by_time_index_wt=m_peers_white.get<by_time>(); - pl_white.resize(pl_white.size() + by_time_index_wt.size()); - for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index_wt)) - { - pl_white.push_back(vl); - } - + peers_indexed::index<by_time>::type& by_time_index = white ? m_peers_white.get<by_time>() : m_peers_gray.get<by_time>(); + for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) + if (!f(vl)) + return false; return true; } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr) + bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed, uint16_t rpc_port) { TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_peerlist_lock); @@ -366,6 +304,8 @@ namespace nodetool ple.adr = addr; ple.id = peer; ple.last_seen = time(NULL); + ple.pruning_seed = pruning_seed; + ple.rpc_port = rpc_port; return append_with_peer_white(ple); CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false); } @@ -469,6 +409,24 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline + bool peerlist_manager::remove_from_peer_white(const peerlist_entry& pe) + { + TRY_ENTRY(); + + CRITICAL_REGION_LOCAL(m_peerlist_lock); + + peers_indexed::index_iterator<by_addr>::type iterator = m_peers_white.get<by_addr>().find(pe.adr); + + if (iterator != m_peers_white.get<by_addr>().end()) { + m_peers_white.erase(iterator); + } + + return true; + + CATCH_ENTRY_L0("peerlist_manager::remove_from_peer_white()", false); + } + //-------------------------------------------------------------------------------------------------- + inline bool peerlist_manager::remove_from_peer_gray(const peerlist_entry& pe) { TRY_ENTRY(); @@ -527,4 +485,3 @@ namespace nodetool //-------------------------------------------------------------------------------------------------- } -BOOST_CLASS_VERSION(nodetool::peerlist_manager, CURRENT_PEERLIST_STORAGE_ARCHIVE_VER) diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index e79207888..40ef2ebcd 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -30,32 +30,59 @@ #pragma once +#include <cstring> + +#include "common/expect.h" #include "net/net_utils_base.h" +#include "net/tor_address.h" +#include "net/i2p_address.h" #include "p2p/p2p_protocol_defs.h" +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED +#include "common/pruning.h" +#endif + +BOOST_CLASS_VERSION(nodetool::peerlist_entry, 2) + namespace boost { namespace serialization { template <class T, class Archive> - inline void do_serialize(Archive &a, epee::net_utils::network_address& na, T local) + inline void do_serialize(boost::mpl::false_, Archive &a, epee::net_utils::network_address& na) + { + T addr{}; + a & addr; + na = std::move(addr); + } + + template <class T, class Archive> + inline void do_serialize(boost::mpl::true_, Archive &a, const epee::net_utils::network_address& na) { - if (typename Archive::is_saving()) local = na.as<T>(); - a & local; - if (!typename Archive::is_saving()) na = local; + a & na.as<T>(); } + template <class Archive, class ver_type> inline void serialize(Archive &a, epee::net_utils::network_address& na, const ver_type ver) { + static constexpr const typename Archive::is_saving is_saving{}; + uint8_t type; - if (typename Archive::is_saving()) - type = na.get_type_id(); + if (is_saving) + type = uint8_t(na.get_type_id()); a & type; - switch (type) + switch (epee::net_utils::address_type(type)) { - case epee::net_utils::ipv4_network_address::ID: - do_serialize(a, na, epee::net_utils::ipv4_network_address{0, 0}); - break; + case epee::net_utils::ipv4_network_address::get_type_id(): + do_serialize<epee::net_utils::ipv4_network_address>(is_saving, a, na); + break; + case net::tor_address::get_type_id(): + do_serialize<net::tor_address>(is_saving, a, na); + break; + case net::i2p_address::get_type_id(): + do_serialize<net::i2p_address>(is_saving, a, na); + break; + case epee::net_utils::address_type::invalid: default: throw std::runtime_error("Unsupported network address type"); } @@ -72,11 +99,113 @@ namespace boost } template <class Archive, class ver_type> + inline void save(Archive& a, const net::tor_address& na, const ver_type) + { + const size_t length = std::strlen(na.host_str()); + if (length > 255) + MONERO_THROW(net::error::invalid_tor_address, "Tor address too long"); + + const uint16_t port{na.port()}; + const uint8_t len = length; + a & port; + a & len; + a.save_binary(na.host_str(), length); + } + + template <class Archive, class ver_type> + inline void save(Archive& a, const net::i2p_address& na, const ver_type) + { + const size_t length = std::strlen(na.host_str()); + if (length > 255) + MONERO_THROW(net::error::invalid_i2p_address, "i2p address too long"); + + const uint16_t port{na.port()}; + const uint8_t len = length; + a & port; + a & len; + a.save_binary(na.host_str(), length); + } + + template <class Archive, class ver_type> + inline void load(Archive& a, net::tor_address& na, const ver_type) + { + uint16_t port = 0; + uint8_t length = 0; + a & port; + a & length; + + if (length > net::tor_address::buffer_size()) + MONERO_THROW(net::error::invalid_tor_address, "Tor address too long"); + + char host[net::tor_address::buffer_size()] = {0}; + a.load_binary(host, length); + host[sizeof(host) - 1] = 0; + + if (std::strcmp(host, net::tor_address::unknown_str()) == 0) + na = net::tor_address::unknown(); + else + na = MONERO_UNWRAP(net::tor_address::make(host, port)); + } + + template <class Archive, class ver_type> + inline void load(Archive& a, net::i2p_address& na, const ver_type) + { + uint16_t port = 0; + uint8_t length = 0; + a & port; + a & length; + + if (length > net::i2p_address::buffer_size()) + MONERO_THROW(net::error::invalid_i2p_address, "i2p address too long"); + + char host[net::i2p_address::buffer_size()] = {0}; + a.load_binary(host, length); + host[sizeof(host) - 1] = 0; + + if (std::strcmp(host, net::i2p_address::unknown_str()) == 0) + na = net::i2p_address::unknown(); + else + na = MONERO_UNWRAP(net::i2p_address::make(host, port)); + } + + template <class Archive, class ver_type> + inline void serialize(Archive &a, net::tor_address& na, const ver_type ver) + { + boost::serialization::split_free(a, na, ver); + } + + template <class Archive, class ver_type> + inline void serialize(Archive &a, net::i2p_address& na, const ver_type ver) + { + boost::serialization::split_free(a, na, ver); + } + + template <class Archive, class ver_type> inline void serialize(Archive &a, nodetool::peerlist_entry& pl, const ver_type ver) { a & pl.adr; a & pl.id; a & pl.last_seen; + if (ver < 1) + { + if (!typename Archive::is_saving()) + pl.pruning_seed = 0; + return; + } + a & pl.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + if (!typename Archive::is_saving()) + { + pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as<epee::net_utils::ipv4_network_address>().ip() % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); + } +#endif + if (ver < 2) + { + if (!typename Archive::is_saving()) + pl.rpc_port = 0; + return; + } + a & pl.rpc_port; } template <class Archive, class ver_type> diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index bb9d2635c..59c6099d5 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -31,8 +31,11 @@ #pragma once #include <boost/uuid/uuid.hpp> +#include <boost/serialization/version.hpp> #include "serialization/keyvalue_serialization.h" #include "net/net_utils_base.h" +#include "net/tor_address.h" // needed for serialization +#include "net/i2p_address.h" // needed for serialization #include "misc_language.h" #include "string_tools.h" #include "time_helper.h" @@ -72,11 +75,15 @@ namespace nodetool AddressType adr; peerid_type id; int64_t last_seen; + uint32_t pruning_seed; + uint16_t rpc_port; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(adr) KV_SERIALIZE(id) KV_SERIALIZE(last_seen) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) + KV_SERIALIZE_OPT(rpc_port, (uint16_t)0) END_KV_SERIALIZE_MAP() }; typedef peerlist_entry_base<epee::net_utils::network_address> peerlist_entry; @@ -122,7 +129,11 @@ namespace nodetool ss << std::setfill ('0') << std::setw (8) << std::hex << std::noshowbase; for(const peerlist_entry& pe: pl) { - ss << pe.id << "\t" << pe.adr.str() << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl; + ss << pe.id << "\t" << pe.adr.str() + << " \trpc port " << (pe.rpc_port > 0 ? std::to_string(pe.rpc_port) : "-") + << " \tpruning seed " << pe.pruning_seed + << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) + << std::endl; } return ss.str(); } @@ -153,6 +164,7 @@ namespace nodetool uuid network_id; uint64_t local_time; uint32_t my_port; + uint16_t rpc_port; peerid_type peer_id; BEGIN_KV_SERIALIZE_MAP() @@ -160,6 +172,7 @@ namespace nodetool KV_SERIALIZE(peer_id) KV_SERIALIZE(local_time) KV_SERIALIZE(my_port) + KV_SERIALIZE_OPT(rpc_port, (uint16_t)(0)) END_KV_SERIALIZE_MAP() }; @@ -174,7 +187,7 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 1; - struct request + struct request_t { basic_node_data node_data; t_playload_type payload_data; @@ -184,8 +197,9 @@ namespace nodetool KV_SERIALIZE(payload_data) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { basic_node_data node_data; t_playload_type payload_data; @@ -201,11 +215,11 @@ namespace nodetool std::vector<peerlist_entry_base<network_address_old>> local_peerlist; for (const auto &p: this_ref.local_peerlist_new) { - if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) { const epee::net_utils::network_address &na = p.adr; const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); - local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen})); + local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen, p.pruning_seed, p.rpc_port})); } else MDEBUG("Not including in legacy peer list: " << p.adr.str()); @@ -220,12 +234,13 @@ namespace nodetool std::vector<peerlist_entry_base<network_address_old>> local_peerlist; epee::serialization::selector<is_store>::serialize_stl_container_pod_val_as_blob(local_peerlist, stg, hparent_section, "local_peerlist"); for (const auto &p: local_peerlist) - ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen})); + ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen, p.pruning_seed, p.rpc_port})); } } END_KV_SERIALIZE_MAP() }; - }; + typedef epee::misc_utils::struct_init<response_t> response; + }; /************************************************************************/ @@ -236,15 +251,16 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 2; - struct request + struct request_t { t_playload_type payload_data; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(payload_data) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t local_time; t_playload_type payload_data; @@ -260,7 +276,7 @@ namespace nodetool std::vector<peerlist_entry_base<network_address_old>> local_peerlist; for (const auto &p: this_ref.local_peerlist_new) { - if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) { const epee::net_utils::network_address &na = p.adr; const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); @@ -284,6 +300,7 @@ namespace nodetool } END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; /************************************************************************/ @@ -301,15 +318,16 @@ namespace nodetool #define PING_OK_RESPONSE_STATUS_TEXT "OK" - struct request + struct request_t { /*actually we don't need to send any real data*/ BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; peerid_type peer_id; @@ -319,6 +337,7 @@ namespace nodetool KV_SERIALIZE(peer_id) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; @@ -345,15 +364,16 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 4; - struct request + struct request_t { proof_of_trust tr; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tr) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string version; std::string os_version; @@ -369,6 +389,7 @@ namespace nodetool KV_SERIALIZE(payload_info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; @@ -379,15 +400,16 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 5; - struct request + struct request_t { proof_of_trust tr; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tr) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::vector<peerlist_entry> local_peerlist_white; std::vector<peerlist_entry> local_peerlist_gray; @@ -402,6 +424,7 @@ namespace nodetool KV_SERIALIZE(local_time) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; /************************************************************************/ @@ -411,13 +434,14 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 6; - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { peerid_type my_id; @@ -425,6 +449,7 @@ namespace nodetool KV_SERIALIZE(my_id) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; /************************************************************************/ @@ -434,13 +459,14 @@ namespace nodetool { const static int ID = P2P_COMMANDS_POOL_BASE + 7; - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint32_t support_flags; @@ -448,6 +474,7 @@ namespace nodetool KV_SERIALIZE(support_flags) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; #endif @@ -462,6 +489,3 @@ namespace nodetool } } - - - diff --git a/src/p2p/stdafx.h b/src/p2p/stdafx.h index b6ff37811..12a371702 100644 --- a/src/p2p/stdafx.h +++ b/src/p2p/stdafx.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/mingw/alloca.h b/src/platform/mingw/alloca.h index 71934b19a..e4722af2a 100644 --- a/src/platform/mingw/alloca.h +++ b/src/platform/mingw/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/alloca.h b/src/platform/msc/alloca.h index 89743e12b..fcf1731b0 100644 --- a/src/platform/msc/alloca.h +++ b/src/platform/msc/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/inline_c.h b/src/platform/msc/inline_c.h index b274f3ec2..23c11bd06 100644 --- a/src/platform/msc/inline_c.h +++ b/src/platform/msc/inline_c.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/stdbool.h b/src/platform/msc/stdbool.h index 63e4200b2..4d24995bd 100644 --- a/src/platform/msc/stdbool.h +++ b/src/platform/msc/stdbool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/sys/param.h b/src/platform/msc/sys/param.h index ca9c9282d..e3cfb7e86 100644 --- a/src/platform/msc/sys/param.h +++ b/src/platform/msc/sys/param.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/ringct/CMakeLists.txt b/src/ringct/CMakeLists.txt index 29f166a3b..0192aa931 100644 --- a/src/ringct/CMakeLists.txt +++ b/src/ringct/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2018, The Monero Project +# Copyright (c) 2016-2019, The Monero Project # # All rights reserved. # diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index d485fb748..e394ef088 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -202,20 +202,36 @@ static rct::keyV vector_powers(const rct::key &x, size_t n) } /* Given a scalar, return the sum of its powers from 0 to n-1 */ -static rct::key vector_power_sum(const rct::key &x, size_t n) +static rct::key vector_power_sum(rct::key x, size_t n) { if (n == 0) return rct::zero(); rct::key res = rct::identity(); if (n == 1) return res; - rct::key prev = x; - for (size_t i = 1; i < n; ++i) + + const bool is_power_of_2 = (n & (n - 1)) == 0; + if (is_power_of_2) { - if (i > 1) - sc_mul(prev.bytes, prev.bytes, x.bytes); - sc_add(res.bytes, res.bytes, prev.bytes); + sc_add(res.bytes, res.bytes, x.bytes); + while (n > 2) + { + sc_mul(x.bytes, x.bytes, x.bytes); + sc_muladd(res.bytes, x.bytes, res.bytes, res.bytes); + n /= 2; + } + } + else + { + rct::key prev = x; + for (size_t i = 1; i < n; ++i) + { + if (i > 1) + sc_mul(prev.bytes, prev.bytes, x.bytes); + sc_add(res.bytes, res.bytes, prev.bytes); + } } + return res; } diff --git a/src/ringct/bulletproofs.h b/src/ringct/bulletproofs.h index b86202ccc..21d494834 100644 --- a/src/ringct/bulletproofs.h +++ b/src/ringct/bulletproofs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/ringct/rctCryptoOps.c b/src/ringct/rctCryptoOps.c index 6fdd17f6b..fbbf6f9bd 100644 --- a/src/ringct/rctCryptoOps.c +++ b/src/ringct/rctCryptoOps.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/ringct/rctCryptoOps.h b/src/ringct/rctCryptoOps.h index e5c1c987a..2a25d13a7 100644 --- a/src/ringct/rctCryptoOps.h +++ b/src/ringct/rctCryptoOps.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 0ec654af6..e39ba16fd 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -670,18 +670,58 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + static key ecdhHash(const key &k) + { + char data[38]; + rct::key hash; + memcpy(data, "amount", 6); + memcpy(data + 6, &k, sizeof(k)); + cn_fast_hash(hash, data, sizeof(data)); + return hash; + } + static void xor8(key &v, const key &k) + { + for (int i = 0; i < 8; ++i) + v.bytes[i] ^= k.bytes[i]; + } + key genCommitmentMask(const key &sk) + { + char data[15 + sizeof(key)]; + memcpy(data, "commitment_mask", 15); + memcpy(data + 15, &sk, sizeof(sk)); + key scalar; + hash_to_scalar(scalar, data, sizeof(data)); + return scalar; + } + + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2) { //encode - sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); - sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + if (v2) + { + unmasked.mask = zero(); + xor8(unmasked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); + sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + } } - void ecdhDecode(ecdhTuple & masked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2) { //decode - sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); - sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + if (v2) + { + masked.mask = genCommitmentMask(sharedSec); + xor8(masked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); + sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + } } } diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 60e920b3a..dd6d87593 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -182,7 +182,8 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec); - void ecdhDecode(ecdhTuple & masked, const key & sharedSec); + key genCommitmentMask(const key &sk); + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2); } #endif /* RCTOPS_H */ diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index baa649f82..25571238e 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -79,9 +79,12 @@ namespace } namespace rct { - Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts, epee::span<const key> sk) { - masks = rct::skvGen(amounts.size()); + CHECK_AND_ASSERT_THROW_MES(amounts.size() == sk.size(), "Invalid amounts/sk sizes"); + masks.resize(amounts.size()); + for (size_t i = 0; i < masks.size(); ++i) + masks[i] = genCommitmentMask(sk[i]); Bulletproof proof = bulletproof_PROVE(amounts, masks); CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); C = proof.V; @@ -416,7 +419,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeBulletproof) + if (rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -466,7 +469,6 @@ namespace rct { //Ver: // verifies the above sig is created corretly mgSig proveRctMG(const key &message, const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, const key &txnFeeKey, hw::device &hwdev) { - mgSig mg; //setup vars size_t cols = pubs.size(); CHECK_AND_ASSERT_THROW_MES(cols >= 1, "Empty pubs"); @@ -524,7 +526,6 @@ namespace rct { // a_out, Cout is for the output commitment // index is the signing index.. mgSig proveRctMGSimple(const key &message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, const multisig_kLRki *kLRki, key *mscout, unsigned int index, hw::device &hwdev) { - mgSig mg; //setup vars size_t rows = 1; size_t cols = pubs.size(); @@ -605,10 +606,19 @@ namespace rct { keyV tmp(rows + 1); size_t i; keyM M(cols, tmp); + ge_p3 Cp3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&Cp3, C.bytes) == 0, false, "point conv failed"); + ge_cached Ccached; + ge_p3_to_cached(&Ccached, &Cp3); + ge_p1p1 p1; //create the matrix to mg sig for (i = 0; i < cols; i++) { M[i][0] = pubs[i].dest; - subKeys(M[i][1], pubs[i].mask, C); + ge_p3 p3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&p3, pubs[i].mask.bytes) == 0, false, "point conv failed"); + ge_sub(&p1, &p3, &Ccached); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_tobytes(M[i][1].bytes, &p3); } //DP(C); return MLSAG_Ver(message, M, mg, rows); @@ -677,7 +687,7 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -707,7 +717,7 @@ namespace rct { //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -728,18 +738,18 @@ namespace rct { return rv; } - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev) { unsigned int index; ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev) { - const bool bulletproof = range_proof_type != RangeProofBorromean; + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { + const bool bulletproof = rct_config.range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -755,7 +765,7 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeBulletproof : RCTTypeSimple; + rv.type = bulletproof ? (rct_config.bp_version == 0 || rct_config.bp_version >= 2 ? RCTTypeBulletproof2 : RCTTypeBulletproof) : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); if (!bulletproof) @@ -781,10 +791,9 @@ namespace rct { rv.p.bulletproofs.clear(); if (bulletproof) { - std::vector<uint64_t> proof_amounts; size_t n_amounts = outamounts.size(); size_t amounts_proved = 0; - if (range_proof_type == RangeProofPaddedBulletproof) + if (rct_config.range_proof_type == RangeProofPaddedBulletproof) { rct::keyV C, masks; if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) @@ -794,7 +803,8 @@ namespace rct { } else { - rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts)); + const epee::span<const key> keys{&amount_keys[0], amount_keys.size()}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts, keys)); #ifdef DBG CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); #endif @@ -808,7 +818,7 @@ namespace rct { else while (amounts_proved < n_amounts) { size_t batch_size = 1; - if (range_proof_type == RangeProofMultiOutputBulletproof) + if (rct_config.range_proof_type == RangeProofMultiOutputBulletproof) while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) batch_size *= 2; rct::keyV C, masks; @@ -822,7 +832,8 @@ namespace rct { } else { - rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts)); + const epee::span<const key> keys{&amount_keys[amounts_proved], batch_size}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts, keys)); #ifdef DBG CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); #endif @@ -844,7 +855,7 @@ namespace rct { //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(outamounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -862,7 +873,6 @@ namespace rct { sc_add(sumpouts.bytes, a[i].bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); } - rv.mixRing = mixRing; sc_sub(a[i].bytes, sumout.bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); DP(pseudoOuts[i]); @@ -876,7 +886,7 @@ namespace rct { return rv; } - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev) { std::vector<unsigned int> index; index.resize(inPk.size()); ctkeyM mixRing; @@ -886,7 +896,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, RangeProofBorromean, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RingCT protocol @@ -976,7 +986,8 @@ namespace rct { { CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); const rctSig &rv = *rvp; - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctSemanticsSimple called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctSemanticsSimple called on non simple rctSig"); const bool bulletproof = is_rct_bulletproof(rv.type); if (bulletproof) { @@ -1075,7 +1086,8 @@ namespace rct { { PERF_TIMER(verRctNonSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctNonSemanticsSimple called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctNonSemanticsSimple called on non simple rctSig"); const bool bulletproof = is_rct_bulletproof(rv.type); // semantics check is early, and mixRing/MGs aren't resolved yet if (bulletproof) @@ -1141,7 +1153,7 @@ namespace rct { //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; @@ -1165,13 +1177,13 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; @@ -1195,7 +1207,7 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index 459edc600..9227eab1e 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -119,10 +119,10 @@ namespace rct { //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev); - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } bool verRctSemanticsSimple(const rctSig & rv); diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 90ed65df0..f01e683cb 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -217,6 +217,7 @@ namespace rct { { case RCTTypeSimple: case RCTTypeBulletproof: + case RCTTypeBulletproof2: return true; default: return false; @@ -228,6 +229,7 @@ namespace rct { switch (type) { case RCTTypeBulletproof: + case RCTTypeBulletproof2: return true; default: return false; diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 487ea6f32..50d0f4d91 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -120,17 +120,14 @@ namespace rct { // If the pedersen commitment to an amount is C = aG + bH, // "mask" contains a 32 byte key a // "amount" contains a hex representation (in 32 bytes) of a 64 bit number - // "senderPk" is not the senders actual public key, but a one-time public key generated for // the purpose of the ECDH exchange struct ecdhTuple { key mask; key amount; - key senderPk; BEGIN_SERIALIZE_OBJECT() - FIELD(mask) + FIELD(mask) // not saved from v2 BPs FIELD(amount) - // FIELD(senderPk) // not serialized, as we do not use it in monero currently END_SERIALIZE() }; @@ -231,8 +228,13 @@ namespace rct { RCTTypeFull = 1, RCTTypeSimple = 2, RCTTypeBulletproof = 3, + RCTTypeBulletproof2 = 4, }; enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; + struct RCTConfig { + RangeProofType range_proof_type; + int bp_version; + }; struct rctSigBase { uint8_t type; key message; @@ -249,7 +251,7 @@ namespace rct { FIELD(type) if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -278,7 +280,19 @@ namespace rct { return false; for (size_t i = 0; i < outputs; ++i) { - FIELDS(ecdhInfo[i]) + if (type == RCTTypeBulletproof2) + { + ar.begin_object(); + if (!typename Archive<W>::is_saving()) + memset(ecdhInfo[i].amount.bytes, 0, sizeof(ecdhInfo[i].amount.bytes)); + crypto::hash8 &amount = (crypto::hash8&)ecdhInfo[i].amount; + FIELD(amount); + ar.end_object(); + } + else + { + FIELDS(ecdhInfo[i]) + } if (outputs - i > 1) ar.delimit_array(); } @@ -310,12 +324,15 @@ namespace rct { { if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; - if (type == RCTTypeBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { uint32_t nbp = bulletproofs.size(); - FIELD(nbp) + if (type == RCTTypeBulletproof2) + VARINT_FIELD(nbp) + else + FIELD(nbp) ar.tag("bp"); ar.begin_array(); if (nbp > outputs) @@ -351,7 +368,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -369,7 +386,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -395,7 +412,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -419,12 +436,12 @@ namespace rct { keyV& get_pseudo_outs() { - return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; } keyV const& get_pseudo_outs() const { - return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; } }; diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index d2c4a33cb..cffe8e1eb 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # @@ -112,6 +112,7 @@ target_link_libraries(rpc common cryptonote_core cryptonote_protocol + net version ${Boost_REGEX_LIBRARY} ${Boost_THREAD_LIBRARY} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0aa25bda7..ff55167fe 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -42,6 +42,7 @@ using namespace epee; #include "cryptonote_basic/account.h" #include "cryptonote_basic/cryptonote_basic_impl.h" #include "misc_language.h" +#include "net/parse.h" #include "storages/http_abstract_invoke.h" #include "crypto/hash.h" #include "rpc/rpc_args.h" @@ -64,6 +65,11 @@ namespace reasons += ", "; reasons += reason; } + + uint64_t round_up(uint64_t value, uint64_t quantum) + { + return (value + quantum - 1) / quantum * quantum; + } } namespace cryptonote @@ -75,6 +81,12 @@ namespace cryptonote command_line::add_arg(desc, arg_rpc_bind_port); command_line::add_arg(desc, arg_rpc_restricted_bind_port); command_line::add_arg(desc, arg_restricted_rpc); + command_line::add_arg(desc, arg_rpc_ssl); + command_line::add_arg(desc, arg_rpc_ssl_private_key); + command_line::add_arg(desc, arg_rpc_ssl_certificate); + command_line::add_arg(desc, arg_rpc_ssl_allowed_certificates); + command_line::add_arg(desc, arg_rpc_ssl_allowed_fingerprints); + command_line::add_arg(desc, arg_rpc_ssl_allow_any_cert); command_line::add_arg(desc, arg_bootstrap_daemon_address); command_line::add_arg(desc, arg_bootstrap_daemon_login); cryptonote::rpc_args::init_options(desc); @@ -111,11 +123,11 @@ namespace cryptonote epee::net_utils::http::login login; login.username = bootstrap_daemon_login.substr(0, loc); login.password = bootstrap_daemon_login.substr(loc + 1); - m_http_client.set_server(m_bootstrap_daemon_address, login, false); + m_http_client.set_server(m_bootstrap_daemon_address, login, epee::net_utils::ssl_support_t::e_ssl_support_autodetect); } else { - m_http_client.set_server(m_bootstrap_daemon_address, boost::none, false); + m_http_client.set_server(m_bootstrap_daemon_address, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_autodetect); } m_should_use_bootstrap_daemon = true; } @@ -130,9 +142,36 @@ namespace cryptonote if (rpc_config->login) http_login.emplace(std::move(rpc_config->login->username), std::move(rpc_config->login->password).password()); + epee::net_utils::ssl_support_t ssl_support; + const std::string ssl = command_line::get_arg(vm, arg_rpc_ssl); + if (!epee::net_utils::ssl_support_from_string(ssl_support, ssl)) + { + MFATAL("Invalid RPC SSL support: " << ssl); + return false; + } + const std::string ssl_private_key = command_line::get_arg(vm, arg_rpc_ssl_private_key); + const std::string ssl_certificate = command_line::get_arg(vm, arg_rpc_ssl_certificate); + const std::vector<std::string> ssl_allowed_certificate_paths = command_line::get_arg(vm, arg_rpc_ssl_allowed_certificates); + std::list<std::string> ssl_allowed_certificates; + for (const std::string &path: ssl_allowed_certificate_paths) + { + ssl_allowed_certificates.push_back({}); + if (!epee::file_io_utils::load_file_to_string(path, ssl_allowed_certificates.back())) + { + MERROR("Failed to load certificate: " << path); + ssl_allowed_certificates.back() = std::string(); + } + } + + const std::vector<std::string> ssl_allowed_fingerprint_strings = command_line::get_arg(vm, arg_rpc_ssl_allowed_fingerprints); + std::vector<std::vector<uint8_t>> ssl_allowed_fingerprints{ ssl_allowed_fingerprint_strings.size() }; + std::transform(ssl_allowed_fingerprint_strings.begin(), ssl_allowed_fingerprint_strings.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); + const bool ssl_allow_any_cert = command_line::get_arg(vm, arg_rpc_ssl_allow_any_cert); + auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; return epee::http_server_impl_base<core_rpc_server, connection_context>::init( - rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login) + rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), + ssl_support, std::make_pair(ssl_private_key, ssl_certificate), std::move(ssl_allowed_certificates), std::move(ssl_allowed_fingerprints), ssl_allow_any_cert ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -147,7 +186,7 @@ namespace cryptonote #define CHECK_CORE_READY() do { if(!check_core_ready()){res.status = CORE_RPC_STATUS_BUSY;return true;} } while(0) //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res) + bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, const connection_context *ctx) { PERF_TIMER(on_get_height); bool r; @@ -159,7 +198,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res) + bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, const connection_context *ctx) { PERF_TIMER(on_get_info); bool r; @@ -173,6 +212,8 @@ namespace cryptonote return r; } + const bool restricted = m_restricted && ctx; + crypto::hash top_hash; m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height @@ -182,13 +223,13 @@ namespace cryptonote res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); - res.alt_blocks_count = m_restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); - uint64_t total_conn = m_restricted ? 0 : m_p2p.get_connections_count(); - res.outgoing_connections_count = m_restricted ? 0 : m_p2p.get_outgoing_connections_count(); - res.incoming_connections_count = m_restricted ? 0 : (total_conn - res.outgoing_connections_count); - res.rpc_connections_count = m_restricted ? 0 : get_connections_count(); - res.white_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_white_peers_count(); - res.grey_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_gray_peers_count(); + res.alt_blocks_count = restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); + uint64_t total_conn = restricted ? 0 : m_p2p.get_public_connections_count(); + res.outgoing_connections_count = restricted ? 0 : m_p2p.get_public_outgoing_connections_count(); + res.incoming_connections_count = restricted ? 0 : (total_conn - res.outgoing_connections_count); + res.rpc_connections_count = restricted ? 0 : get_connections_count(); + res.white_peerlist_size = restricted ? 0 : m_p2p.get_public_white_peers_count(); + res.grey_peerlist_size = restricted ? 0 : m_p2p.get_public_gray_peers_count(); cryptonote::network_type net_type = nettype(); res.mainnet = net_type == MAINNET; @@ -199,21 +240,23 @@ namespace cryptonote res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; - res.start_time = m_restricted ? 0 : (uint64_t)m_core.get_start_time(); - res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); + res.start_time = restricted ? 0 : (uint64_t)m_core.get_start_time(); + res.free_space = restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); res.offline = m_core.offline(); - res.bootstrap_daemon_address = m_restricted ? "" : m_bootstrap_daemon_address; - res.height_without_bootstrap = m_restricted ? 0 : res.height; - if (m_restricted) + res.bootstrap_daemon_address = restricted ? "" : m_bootstrap_daemon_address; + res.height_without_bootstrap = restricted ? 0 : res.height; + if (restricted) res.was_bootstrap_ever_used = false; else { boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } - res.database_size = m_restricted ? 0 : m_core.get_blockchain_storage().get_db().get_database_size(); - res.update_available = m_restricted ? false : m_core.is_update_available(); - res.version = m_restricted ? "" : MONERO_VERSION; + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); + if (restricted) + res.database_size = round_up(res.database_size, 5ull* 1024 * 1024 * 1024); + res.update_available = restricted ? false : m_core.is_update_available(); + res.version = restricted ? "" : MONERO_VERSION; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -227,7 +270,7 @@ namespace cryptonote END_SERIALIZE() }; //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) + bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, const connection_context *ctx) { PERF_TIMER(on_get_blocks); bool r; @@ -252,19 +295,11 @@ namespace cryptonote pruned_size += bd.first.first.size(); unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - if (!req.no_miner_tx) - { - bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices); - if (!r) - { - res.status = "Failed"; - return false; - } - } ntxes += bd.second.size(); + res.output_indices.back().indices.reserve(1 + bd.second.size()); + if (req.no_miner_tx) + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); res.blocks.back().txs.reserve(bd.second.size()); - res.output_indices.back().indices.reserve(bd.second.size()); for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { unpruned_size += i->second.size(); @@ -272,14 +307,25 @@ namespace cryptonote i->second.clear(); i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); + } - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); + const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); + if (n_txes_to_lookup > 0) + { + std::vector<std::vector<uint64_t>> indices; + bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); if (!r) { res.status = "Failed"; return false; } + if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + { + res.status = "Failed"; + return false; + } + for (size_t i = 0; i < indices.size(); ++i) + res.output_indices.back().indices.push_back({std::move(indices[i])}); } } @@ -287,7 +333,7 @@ namespace cryptonote res.status = CORE_RPC_STATUS_OK; return true; } - bool core_rpc_server::on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res) + bool core_rpc_server::on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res, const connection_context *ctx) { PERF_TIMER(on_get_alt_blocks_hashes); bool r; @@ -314,7 +360,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res) + bool core_rpc_server::on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res, const connection_context *ctx) { PERF_TIMER(on_get_blocks_by_height); bool r; @@ -348,7 +394,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res) + bool core_rpc_server::on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res, const connection_context *ctx) { PERF_TIMER(on_get_hashes); bool r; @@ -371,7 +417,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) + bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res, const connection_context *ctx) { PERF_TIMER(on_get_outs_bin); bool r; @@ -380,7 +426,8 @@ namespace cryptonote res.status = "Failed"; - if (m_restricted) + const bool restricted = m_restricted && ctx; + if (restricted) { if (req.outputs.size() > MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT) { @@ -398,7 +445,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res) + bool core_rpc_server::on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res, const connection_context *ctx) { PERF_TIMER(on_get_outs); bool r; @@ -407,7 +454,8 @@ namespace cryptonote res.status = "Failed"; - if (m_restricted) + const bool restricted = m_restricted && ctx; + if (restricted) { if (req.outputs.size() > MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT) { @@ -440,7 +488,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) + bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, const connection_context *ctx) { PERF_TIMER(on_get_indexes); bool ok; @@ -458,7 +506,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res) + bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transactions); bool ok; @@ -482,8 +530,8 @@ namespace cryptonote vh.push_back(*reinterpret_cast<const crypto::hash*>(b.data())); } std::vector<crypto::hash> missed_txs; - std::vector<transaction> txs; - bool r = m_core.get_transactions(vh, txs, missed_txs); + std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> txs; + bool r = m_core.get_split_transactions_blobs(vh, txs, missed_txs); if(!r) { res.status = "Failed"; @@ -503,7 +551,7 @@ namespace cryptonote if(r) { // sort to match original request - std::vector<transaction> sorted_txs; + std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs; std::vector<tx_info>::const_iterator i; unsigned txs_processed = 0; for (const crypto::hash &h: vh) @@ -516,7 +564,7 @@ namespace cryptonote return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs[txs_processed]) != h) + if (std::get<0>(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; @@ -532,7 +580,16 @@ namespace cryptonote res.status = "Failed to parse and validate tx from blob"; return true; } - sorted_txs.push_back(tx); + std::stringstream ss; + binary_archive<true> ba(ss); + bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); + if (!r) + { + res.status = "Failed to serialize transaction base"; + return true; + } + const cryptonote::blobdata pruned = ss.str(); + sorted_txs.push_back(std::make_tuple(h, pruned, get_transaction_prunable_hash(tx), std::string(i->tx_blob, pruned.size()))); missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); @@ -561,11 +618,36 @@ namespace cryptonote crypto::hash tx_hash = *vhi++; e.tx_hash = *txhi++; - pruned_transaction pruned_tx{tx}; - blobdata blob = req.prune ? t_serializable_object_to_blob(pruned_tx) : t_serializable_object_to_blob(tx); - e.as_hex = string_tools::buff_to_hex_nodelimer(blob); - if (req.decode_as_json) - e.as_json = req.prune ? obj_to_json_str(pruned_tx) : obj_to_json_str(tx); + e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); + if (req.split || req.prune || std::get<3>(tx).empty()) + { + e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx)); + if (!req.prune) + e.prunable_as_hex = string_tools::buff_to_hex_nodelimer(std::get<3>(tx)); + } + else + { + cryptonote::blobdata tx_data; + if (req.prune) + tx_data = std::get<1>(tx); + else + tx_data = std::get<1>(tx) + std::get<3>(tx); + e.as_hex = string_tools::buff_to_hex_nodelimer(tx_data); + if (req.decode_as_json && !tx_data.empty()) + { + cryptonote::transaction t; + if (cryptonote::parse_and_validate_tx_from_blob(tx_data, t)) + { + if (req.prune) + { + pruned_transaction pruned_tx{t}; + e.as_json = obj_to_json_str(pruned_tx); + } + else + e.as_json = obj_to_json_str(t); + } + } + } e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end(); if (e.in_pool) { @@ -614,13 +696,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, const connection_context *ctx) { PERF_TIMER(on_is_key_image_spent); bool ok; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_IS_KEY_IMAGE_SPENT>(invoke_http_mode::JON, "/is_key_image_spent", req, res, ok)) return ok; + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; std::vector<crypto::key_image> key_images; for(const auto& ki_hex_str: req.key_images) { @@ -650,7 +734,7 @@ namespace cryptonote // check the pool too std::vector<cryptonote::tx_info> txs; std::vector<cryptonote::spent_key_image_info> ki; - r = m_core.get_pool_transactions_and_spent_keys_info(txs, ki, !request_has_rpc_origin || !m_restricted); + r = m_core.get_pool_transactions_and_spent_keys_info(txs, ki, !request_has_rpc_origin || !restricted); if(!r) { res.status = "Failed"; @@ -681,7 +765,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res) + bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, const connection_context *ctx) { PERF_TIMER(on_send_raw_tx); bool ok; @@ -703,31 +787,31 @@ namespace cryptonote if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) { res.status = "Failed"; - res.reason = ""; + std::string reason = ""; if ((res.low_mixin = tvc.m_low_mixin)) - add_reason(res.reason, "bad ring size"); + add_reason(reason, "bad ring size"); if ((res.double_spend = tvc.m_double_spend)) - add_reason(res.reason, "double spend"); + add_reason(reason, "double spend"); if ((res.invalid_input = tvc.m_invalid_input)) - add_reason(res.reason, "invalid input"); + add_reason(reason, "invalid input"); if ((res.invalid_output = tvc.m_invalid_output)) - add_reason(res.reason, "invalid output"); + add_reason(reason, "invalid output"); if ((res.too_big = tvc.m_too_big)) - add_reason(res.reason, "too big"); + add_reason(reason, "too big"); if ((res.overspend = tvc.m_overspend)) - add_reason(res.reason, "overspend"); + add_reason(reason, "overspend"); if ((res.fee_too_low = tvc.m_fee_too_low)) - add_reason(res.reason, "fee too low"); + add_reason(reason, "fee too low"); if ((res.not_rct = tvc.m_not_rct)) - add_reason(res.reason, "tx is not ringct"); - const std::string punctuation = res.reason.empty() ? "" : ": "; + add_reason(reason, "tx is not ringct"); + const std::string punctuation = reason.empty() ? "" : ": "; if (tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << reason); } else { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << reason); } return true; } @@ -749,7 +833,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res) + bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, const connection_context *ctx) { PERF_TIMER(on_start_mining); CHECK_CORE_READY(); @@ -803,7 +887,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res) + bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, const connection_context *ctx) { PERF_TIMER(on_stop_mining); cryptonote::miner &miner= m_core.get_miner(); @@ -823,7 +907,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res) + bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, const connection_context *ctx) { PERF_TIMER(on_mining_status); @@ -831,18 +915,37 @@ namespace cryptonote res.active = lMiner.is_mining(); res.is_background_mining_enabled = lMiner.get_is_background_mining_enabled(); + res.block_target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; if ( lMiner.is_mining() ) { res.speed = lMiner.get_speed(); res.threads_count = lMiner.get_threads_count(); - const account_public_address& lMiningAdr = lMiner.get_mining_address(); - res.address = get_account_address_as_str(nettype(), false, lMiningAdr); + res.block_reward = lMiner.get_block_reward(); + } + const account_public_address& lMiningAdr = lMiner.get_mining_address(); + res.address = get_account_address_as_str(nettype(), false, lMiningAdr); + const uint8_t major_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); + const unsigned variant = major_version >= 7 ? major_version - 6 : 0; + switch (variant) + { + case 0: res.pow_algorithm = "Cryptonight"; break; + case 1: res.pow_algorithm = "CNv1 (Cryptonight variant 1)"; break; + case 2: case 3: res.pow_algorithm = "CNv2 (Cryptonight variant 2)"; break; + case 4: case 5: res.pow_algorithm = "CNv4 (Cryptonight variant 4)"; break; + default: res.pow_algorithm = "I'm not sure actually"; break; + } + if (res.is_background_mining_enabled) + { + res.bg_idle_threshold = lMiner.get_idle_threshold(); + res.bg_min_idle_seconds = lMiner.get_min_idle_seconds(); + res.bg_ignore_battery = lMiner.get_ignore_battery(); + res.bg_target = lMiner.get_mining_target(); } res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res) + bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, const connection_context *ctx) { PERF_TIMER(on_save_bc); if( !m_core.get_blockchain_storage().store_blockchain() ) @@ -854,38 +957,38 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res) + bool core_rpc_server::on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res, const connection_context *ctx) { PERF_TIMER(on_get_peer_list); std::vector<nodetool::peerlist_entry> white_list; std::vector<nodetool::peerlist_entry> gray_list; - m_p2p.get_peerlist_manager().get_peerlist_full(gray_list, white_list); + m_p2p.get_public_peerlist(gray_list, white_list); res.white_list.reserve(white_list.size()); for (auto & entry : white_list) { - if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) res.white_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), - entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen); + entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); else - res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed, entry.rpc_port); } res.gray_list.reserve(gray_list.size()); for (auto & entry : gray_list) { - if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) + if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) res.gray_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), - entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen); + entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port); else - res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed, entry.rpc_port); } res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res) + bool core_rpc_server::on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res, const connection_context *ctx) { PERF_TIMER(on_set_log_hash_rate); if(m_core.get_miner().is_mining()) @@ -900,7 +1003,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res) + bool core_rpc_server::on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res, const connection_context *ctx) { PERF_TIMER(on_set_log_level); if (req.level < 0 || req.level > 4) @@ -913,7 +1016,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res) + bool core_rpc_server::on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res, const connection_context *ctx) { PERF_TIMER(on_set_log_categories); mlog_set_log(req.categories.c_str()); @@ -922,41 +1025,47 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transaction_pool); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL>(invoke_http_mode::JON, "/get_transaction_pool", req, res, r)) return r; - m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, !request_has_rpc_origin || !m_restricted); + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, !request_has_rpc_origin || !restricted); for (tx_info& txi : res.transactions) txi.tx_blob = epee::string_tools::buff_to_hex_nodelimer(txi.tx_blob); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transaction_pool_hashes); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN>(invoke_http_mode::JON, "/get_transaction_pool_hashes.bin", req, res, r)) return r; - m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !m_restricted); + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !restricted); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transaction_pool_hashes); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES>(invoke_http_mode::JON, "/get_transaction_pool_hashes", req, res, r)) return r; + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; std::vector<crypto::hash> tx_hashes; - m_core.get_pool_transaction_hashes(tx_hashes, !request_has_rpc_origin || !m_restricted); + m_core.get_pool_transaction_hashes(tx_hashes, !request_has_rpc_origin || !restricted); res.tx_hashes.reserve(tx_hashes.size()); for (const crypto::hash &tx_hash: tx_hashes) res.tx_hashes.push_back(epee::string_tools::pod_to_hex(tx_hash)); @@ -964,19 +1073,21 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, const connection_context *ctx) { PERF_TIMER(on_get_transaction_pool_stats); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_STATS>(invoke_http_mode::JON, "/get_transaction_pool_stats", req, res, r)) return r; - m_core.get_pool_transaction_stats(res.pool_stats, !request_has_rpc_origin || !m_restricted); + const bool restricted = m_restricted && ctx; + const bool request_has_rpc_origin = ctx != NULL; + m_core.get_pool_transaction_stats(res.pool_stats, !request_has_rpc_origin || !restricted); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res) + bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res, const connection_context *ctx) { PERF_TIMER(on_stop_daemon); // FIXME: replace back to original m_p2p.send_stop_signal() after @@ -986,7 +1097,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res) + bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx) { PERF_TIMER(on_getblockcount); { @@ -1002,7 +1113,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_getblockhash); { @@ -1046,7 +1157,7 @@ namespace cryptonote return 0; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_getblocktemplate); bool r; @@ -1125,7 +1236,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_submitblock); { @@ -1180,7 +1291,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_generateblocks); @@ -1209,7 +1320,7 @@ namespace cryptonote for(size_t i = 0; i < req.amount_of_blocks; i++) { - r = on_getblocktemplate(template_req, template_res, error_resp); + r = on_getblocktemplate(template_req, template_res, error_resp, ctx); res.status = template_res.status; if (!r) return false; @@ -1231,7 +1342,7 @@ namespace cryptonote miner::find_nonce_for_given_block(b, template_res.difficulty, template_res.height); submit_req.front() = string_tools::buff_to_hex_nodelimer(block_to_blob(b)); - r = on_submitblock(submit_req, submit_res, error_resp); + r = on_submitblock(submit_req, submit_res, error_resp, ctx); res.status = submit_res.status; if (!r) return false; @@ -1270,6 +1381,7 @@ namespace cryptonote response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; + response.long_term_weight = m_core.get_blockchain_storage().get_db().get_block_long_term_weight(height); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1340,7 +1452,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_last_block_header); bool r; @@ -1370,7 +1482,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block_header_by_hash); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH>(invoke_http_mode::JON_RPC, "getblockheaderbyhash", req, res, r)) @@ -1411,7 +1524,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block_headers_range); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCK_HEADERS_RANGE>(invoke_http_mode::JON_RPC, "getblockheadersrange", req, res, r)) @@ -1461,7 +1575,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block_header_by_height); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT>(invoke_http_mode::JON_RPC, "getblockheaderbyheight", req, res, r)) @@ -1493,7 +1608,8 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp){ + bool core_rpc_server::on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { PERF_TIMER(on_get_block); bool r; if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCK>(invoke_http_mode::JON_RPC, "getblock", req, res, r)) @@ -1554,7 +1670,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_connections); @@ -1565,7 +1681,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_info_json); bool r; @@ -1579,6 +1695,8 @@ namespace cryptonote return r; } + const bool restricted = m_restricted && ctx; + crypto::hash top_hash; m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height @@ -1588,13 +1706,13 @@ namespace cryptonote res.target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); - res.alt_blocks_count = m_restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); - uint64_t total_conn = m_restricted ? 0 : m_p2p.get_connections_count(); - res.outgoing_connections_count = m_restricted ? 0 : m_p2p.get_outgoing_connections_count(); - res.incoming_connections_count = m_restricted ? 0 : (total_conn - res.outgoing_connections_count); - res.rpc_connections_count = m_restricted ? 0 : get_connections_count(); - res.white_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_white_peers_count(); - res.grey_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_gray_peers_count(); + res.alt_blocks_count = restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); + uint64_t total_conn = restricted ? 0 : m_p2p.get_public_connections_count(); + res.outgoing_connections_count = restricted ? 0 : m_p2p.get_public_outgoing_connections_count(); + res.incoming_connections_count = restricted ? 0 : (total_conn - res.outgoing_connections_count); + res.rpc_connections_count = restricted ? 0 : get_connections_count(); + res.white_peerlist_size = restricted ? 0 : m_p2p.get_public_white_peers_count(); + res.grey_peerlist_size = restricted ? 0 : m_p2p.get_public_gray_peers_count(); cryptonote::network_type net_type = nettype(); res.mainnet = net_type == MAINNET; @@ -1606,25 +1724,25 @@ namespace cryptonote res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; - res.start_time = m_restricted ? 0 : (uint64_t)m_core.get_start_time(); - res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); + res.start_time = restricted ? 0 : (uint64_t)m_core.get_start_time(); + res.free_space = restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); res.offline = m_core.offline(); - res.bootstrap_daemon_address = m_restricted ? "" : m_bootstrap_daemon_address; - res.height_without_bootstrap = m_restricted ? 0 : res.height; - if (m_restricted) + res.bootstrap_daemon_address = restricted ? "" : m_bootstrap_daemon_address; + res.height_without_bootstrap = restricted ? 0 : res.height; + if (restricted) res.was_bootstrap_ever_used = false; else { boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } - res.database_size = m_restricted ? 0 : m_core.get_blockchain_storage().get_db().get_database_size(); - res.update_available = m_restricted ? false : m_core.is_update_available(); - res.version = m_restricted ? "" : MONERO_VERSION; + res.database_size = restricted ? 0 : m_core.get_blockchain_storage().get_db().get_database_size(); + res.update_available = restricted ? false : m_core.is_update_available(); + res.version = restricted ? "" : MONERO_VERSION; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_hard_fork_info); bool r; @@ -1640,7 +1758,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_bans); @@ -1664,7 +1782,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_set_bans); @@ -1673,12 +1791,14 @@ namespace cryptonote epee::net_utils::network_address na; if (!i->host.empty()) { - if (!epee::net_utils::create_network_address(na, i->host)) + auto na_parsed = net::get_network_address(i->host, 0); + if (!na_parsed) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; error_resp.message = "Unsupported host type"; return false; } + na = std::move(*na_parsed); } else { @@ -1694,7 +1814,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_flush_txpool); @@ -1749,7 +1869,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_output_histogram); bool r; @@ -1779,7 +1899,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_version); bool r; @@ -1791,7 +1911,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_coinbase_tx_sum); std::pair<uint64_t, uint64_t> amounts = m_core.get_coinbase_tx_sum(req.height, req.count); @@ -1801,7 +1921,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_base_fee_estimate); bool r; @@ -1814,7 +1934,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_alternate_chains); try @@ -1845,7 +1965,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res) + bool core_rpc_server::on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res, const connection_context *ctx) { PERF_TIMER(on_get_limit); bool r; @@ -1858,7 +1978,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res) + bool core_rpc_server::on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res, const connection_context *ctx) { PERF_TIMER(on_set_limit); // -1 = reset to default @@ -1898,31 +2018,23 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res) + bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res, const connection_context *ctx) { PERF_TIMER(on_out_peers); - size_t n_connections = m_p2p.get_outgoing_connections_count(); - size_t n_delete = (n_connections > req.out_peers) ? n_connections - req.out_peers : 0; - m_p2p.m_config.m_net_config.max_out_connection_count = req.out_peers; - if (n_delete) - m_p2p.delete_out_connections(n_delete); + m_p2p.change_max_out_public_peers(req.out_peers); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res) + bool core_rpc_server::on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res, const connection_context *ctx) { PERF_TIMER(on_in_peers); - size_t n_connections = m_p2p.get_incoming_connections_count(); - size_t n_delete = (n_connections > req.in_peers) ? n_connections - req.in_peers : 0; - m_p2p.m_config.m_net_config.max_in_connection_count = req.in_peers; - if (n_delete) - m_p2p.delete_in_connections(n_delete); + m_p2p.change_max_in_public_peers(req.in_peers); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res) + bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res, const connection_context *ctx) { PERF_TIMER(on_start_save_graph); m_p2p.set_save_graph(true); @@ -1930,7 +2042,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res) + bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res, const connection_context *ctx) { PERF_TIMER(on_stop_save_graph); m_p2p.set_save_graph(false); @@ -1938,7 +2050,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res) + bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res, const connection_context *ctx) { PERF_TIMER(on_update); static const char software[] = "monero"; @@ -2033,7 +2145,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res) + bool core_rpc_server::on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res, const connection_context *ctx) { PERF_TIMER(on_pop_blocks); @@ -2045,7 +2157,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_relay_tx); @@ -2091,7 +2203,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_sync_info); @@ -2099,6 +2211,7 @@ namespace cryptonote m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.target_height = m_core.get_target_blockchain_height(); + res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second; for (const auto &c: m_p2p.get_payload_object().get_connections()) res.peers.push_back({c}); @@ -2113,12 +2226,13 @@ namespace cryptonote res.spans.push_back({span.start_block_height, span.nblocks, span_connection_id, (uint32_t)(span.rate + 0.5f), speed, span.size, address}); return true; }); + res.overview = block_queue.get_overview(res.height); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_txpool_backlog); bool r; @@ -2136,7 +2250,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { PERF_TIMER(on_get_output_distribution); bool r; @@ -2171,7 +2285,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res) + bool core_rpc_server::on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, const connection_context *ctx) { PERF_TIMER(on_get_output_distribution_bin); @@ -2212,6 +2326,29 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + try + { + if (!(req.check ? m_core.check_blockchain_pruning() : m_core.prune_blockchain())) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = req.check ? "Failed to check blockchain pruning" : "Failed to prune blockchain"; + return false; + } + res.pruning_seed = m_core.get_blockchain_pruning_seed(); + } + catch (const std::exception &e) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to prune blockchain"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ const command_line::arg_descriptor<std::string, false, true, 2> core_rpc_server::arg_rpc_bind_port = { @@ -2240,6 +2377,40 @@ namespace cryptonote , false }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl = { + "rpc-ssl" + , "Enable SSL on RPC connections: enabled|disabled|autodetect" + , "autodetect" + }; + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_private_key = { + "rpc-ssl-private-key" + , "Path to a PEM format private key" + , "" + }; + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_certificate = { + "rpc-ssl-certificate" + , "Path to a PEM format certificate" + , "" + }; + + const command_line::arg_descriptor<std::vector<std::string>> core_rpc_server::arg_rpc_ssl_allowed_certificates = { + "rpc-ssl-allowed-certificates" + , "List of paths to PEM format certificates of allowed peers (all allowed if empty)" + }; + + const command_line::arg_descriptor<std::vector<std::string>> core_rpc_server::arg_rpc_ssl_allowed_fingerprints = { + "rpc-ssl-allowed-fingerprints" + , "List of certificate fingerprints to allow" + }; + + const command_line::arg_descriptor<bool> core_rpc_server::arg_rpc_ssl_allow_any_cert = { + "rpc-ssl-allow-any-cert" + , "Allow any peer certificate, rather than just those on the allowed list" + , false + }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_bootstrap_daemon_address = { "bootstrap-daemon-address" , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 081ccc25d..fe066b31b 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -53,9 +53,16 @@ namespace cryptonote { public: + static const command_line::arg_descriptor<bool> arg_public_node; static const command_line::arg_descriptor<std::string, false, true, 2> arg_rpc_bind_port; static const command_line::arg_descriptor<std::string> arg_rpc_restricted_bind_port; static const command_line::arg_descriptor<bool> arg_restricted_rpc; + static const command_line::arg_descriptor<std::string> arg_rpc_ssl; + static const command_line::arg_descriptor<std::string> arg_rpc_ssl_private_key; + static const command_line::arg_descriptor<std::string> arg_rpc_ssl_certificate; + static const command_line::arg_descriptor<std::vector<std::string>> arg_rpc_ssl_allowed_certificates; + static const command_line::arg_descriptor<std::vector<std::string>> arg_rpc_ssl_allowed_fingerprints; + static const command_line::arg_descriptor<bool> arg_rpc_ssl_allow_any_cert; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_address; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_login; @@ -153,70 +160,72 @@ namespace cryptonote MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) MAP_JON_RPC_WE("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG) MAP_JON_RPC_WE("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) + MAP_JON_RPC_WE_IF("prune_blockchain", on_prune_blockchain, COMMAND_RPC_PRUNE_BLOCKCHAIN, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() - bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res); - bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res); - bool on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res); - bool on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res); - bool on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res); - bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res); - bool on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, bool request_has_rpc_origin = true); - bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res); - bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res); - bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); - bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); - bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); - bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res); - bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res); - bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); - bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); - bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); - bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res); - bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); - bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res); - bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, bool request_has_rpc_origin = true); - bool on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, bool request_has_rpc_origin = true); - bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin = true); - bool on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin = true); - bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); - bool on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res); - bool on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res); - bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res); - bool on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res); - bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); - bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); - bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res); - bool on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res); - bool on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res); + bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, const connection_context *ctx = NULL); + bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, const connection_context *ctx = NULL); + bool on_get_alt_blocks_hashes(const COMMAND_RPC_GET_ALT_BLOCKS_HASHES::request& req, COMMAND_RPC_GET_ALT_BLOCKS_HASHES::response& res, const connection_context *ctx = NULL); + bool on_get_blocks_by_height(const COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCKS_BY_HEIGHT::response& res, const connection_context *ctx = NULL); + bool on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res, const connection_context *ctx = NULL); + bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, const connection_context *ctx = NULL); + bool on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res, const connection_context *ctx = NULL); + bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, const connection_context *ctx = NULL); + bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, const connection_context *ctx = NULL); + bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, const connection_context *ctx = NULL); + bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, const connection_context *ctx = NULL); + bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, const connection_context *ctx = NULL); + bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res, const connection_context *ctx = NULL); + bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res, const connection_context *ctx = NULL); + bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, const connection_context *ctx = NULL); + bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, const connection_context *ctx = NULL); + bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res, const connection_context *ctx = NULL); + bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res, const connection_context *ctx = NULL); + bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res, const connection_context *ctx = NULL); + bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, const connection_context *ctx = NULL); + bool on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, const connection_context *ctx = NULL); + bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res, const connection_context *ctx = NULL); + bool on_get_limit(const COMMAND_RPC_GET_LIMIT::request& req, COMMAND_RPC_GET_LIMIT::response& res, const connection_context *ctx = NULL); + bool on_set_limit(const COMMAND_RPC_SET_LIMIT::request& req, COMMAND_RPC_SET_LIMIT::response& res, const connection_context *ctx = NULL); + bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res, const connection_context *ctx = NULL); + bool on_in_peers(const COMMAND_RPC_IN_PEERS::request& req, COMMAND_RPC_IN_PEERS::response& res, const connection_context *ctx = NULL); + bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res, const connection_context *ctx = NULL); + bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res, const connection_context *ctx = NULL); + bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res, const connection_context *ctx = NULL); + bool on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, const connection_context *ctx = NULL); + bool on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res, const connection_context *ctx = NULL); //json_rpc - bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); - bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp); - bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp); - bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp); - bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp); - bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp); - bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp); - bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp); - bool on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp); - bool on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp); - bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp); - bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp); - bool on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp); - bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp); - bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp); - bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp); - bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp); - bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp); - bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp); - bool on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); - bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); - bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); - bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); - bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); - bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp); + bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); + bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block_headers_range(const COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request& req, COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_block(const COMMAND_RPC_GET_BLOCK::request& req, COMMAND_RPC_GET_BLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 0a07930ec..f65c7c8dd 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -84,19 +84,20 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 2 +#define CORE_RPC_VERSION_MINOR 4 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) struct COMMAND_RPC_GET_HEIGHT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t height; std::string status; @@ -108,12 +109,13 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BLOCKS_FAST { - struct request + struct request_t { std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; @@ -126,6 +128,7 @@ namespace cryptonote KV_SERIALIZE_OPT(no_miner_tx, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct tx_output_indices { @@ -145,7 +148,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::vector<block_complete_entry> blocks; uint64_t start_height; @@ -163,19 +166,21 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BLOCKS_BY_HEIGHT { - struct request + struct request_t { std::vector<uint64_t> heights; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(heights) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::vector<block_complete_entry> blocks; std::string status; @@ -187,17 +192,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ALT_BLOCKS_HASHES { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::vector<std::string> blks_hashes; std::string status; @@ -209,11 +216,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_HASHES_FAST { - struct request + struct request_t { std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; @@ -222,8 +230,9 @@ namespace cryptonote KV_SERIALIZE(start_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::vector<crypto::hash> m_block_ids; uint64_t start_height; @@ -239,12 +248,13 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_ADDRESS_TXS { - struct request + struct request_t { std::string address; std::string view_key; @@ -254,6 +264,7 @@ namespace cryptonote KV_SERIALIZE(view_key) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct spent_output { uint64_t amount; @@ -304,7 +315,7 @@ namespace cryptonote }; - struct response + struct response_t { //std::list<std::string> txs_as_json; uint64_t total_received; @@ -324,12 +335,13 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_ADDRESS_INFO { - struct request + struct request_t { std::string address; std::string view_key; @@ -339,6 +351,7 @@ namespace cryptonote KV_SERIALIZE(view_key) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct spent_output { @@ -356,10 +369,8 @@ namespace cryptonote KV_SERIALIZE(mixin) END_KV_SERIALIZE_MAP() }; - - - - struct response + + struct response_t { uint64_t locked_funds; uint64_t total_received; @@ -382,12 +393,13 @@ namespace cryptonote KV_SERIALIZE(spent_outputs) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_UNSPENT_OUTS { - struct request + struct request_t { std::string amount; std::string address; @@ -406,6 +418,7 @@ namespace cryptonote KV_SERIALIZE(dust_threshold) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct output { @@ -437,7 +450,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { uint64_t amount; std::list<output> outputs; @@ -452,12 +465,13 @@ namespace cryptonote KV_SERIALIZE(reason) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_RANDOM_OUTS { - struct request + struct request_t { std::vector<std::string> amounts; uint32_t count; @@ -467,6 +481,7 @@ namespace cryptonote KV_SERIALIZE(count) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct output { @@ -491,7 +506,7 @@ namespace cryptonote }; - struct response + struct response_t { std::vector<amount_out> amount_outs; std::string Error; @@ -500,11 +515,12 @@ namespace cryptonote KV_SERIALIZE(Error) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_SUBMIT_RAW_TX { - struct request + struct request_t { std::string address; std::string view_key; @@ -516,9 +532,10 @@ namespace cryptonote KV_SERIALIZE(tx) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::string error; @@ -528,11 +545,12 @@ namespace cryptonote KV_SERIALIZE(error) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_LOGIN { - struct request + struct request_t { std::string address; std::string view_key; @@ -544,9 +562,9 @@ namespace cryptonote KV_SERIALIZE(create_account) END_KV_SERIALIZE_MAP() }; - - - struct response + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t { std::string status; std::string reason; @@ -558,11 +576,12 @@ namespace cryptonote KV_SERIALIZE(new_address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_IMPORT_WALLET_REQUEST { - struct request + struct request_t { std::string address; std::string view_key; @@ -572,9 +591,9 @@ namespace cryptonote KV_SERIALIZE(view_key) END_KV_SERIALIZE_MAP() }; - - - struct response + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t { std::string payment_id; uint64_t import_fee; @@ -592,27 +611,34 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_TRANSACTIONS { - struct request + struct request_t { std::vector<std::string> txs_hashes; bool decode_as_json; bool prune; + bool split; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs_hashes) KV_SERIALIZE(decode_as_json) KV_SERIALIZE_OPT(prune, false) + KV_SERIALIZE_OPT(split, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct entry { std::string tx_hash; std::string as_hex; + std::string pruned_as_hex; + std::string prunable_as_hex; + std::string prunable_hash; std::string as_json; bool in_pool; bool double_spend_seen; @@ -623,6 +649,9 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(as_hex) + KV_SERIALIZE(pruned_as_hex) + KV_SERIALIZE(prunable_as_hex) + KV_SERIALIZE(prunable_hash) KV_SERIALIZE(as_json) KV_SERIALIZE(in_pool) KV_SERIALIZE(double_spend_seen) @@ -632,7 +661,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { // older compatibility stuff std::vector<std::string> txs_as_hex; //transactions blobs as hex (old compat) @@ -655,6 +684,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- @@ -666,7 +696,7 @@ namespace cryptonote SPENT_IN_POOL = 2, }; - struct request + struct request_t { std::vector<std::string> key_images; @@ -674,9 +704,10 @@ namespace cryptonote KV_SERIALIZE(key_images) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::vector<int> spent_status; std::string status; @@ -688,21 +719,23 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES { - struct request + struct request_t { crypto::hash txid; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_VAL_POD_AS_BLOB(txid) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::vector<uint64_t> o_indexes; std::string status; @@ -713,6 +746,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct get_outputs_out @@ -728,7 +762,7 @@ namespace cryptonote struct COMMAND_RPC_GET_OUTPUTS_BIN { - struct request + struct request_t { std::vector<get_outputs_out> outputs; bool get_txid; @@ -738,6 +772,7 @@ namespace cryptonote KV_SERIALIZE_OPT(get_txid, true) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct outkey { @@ -756,7 +791,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::vector<outkey> outs; std::string status; @@ -768,11 +803,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_OUTPUTS { - struct request + struct request_t { std::vector<get_outputs_out> outputs; @@ -780,6 +816,7 @@ namespace cryptonote KV_SERIALIZE(outputs) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct outkey { @@ -798,7 +835,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::vector<outkey> outs; std::string status; @@ -810,11 +847,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_SEND_RAW_TX { - struct request + struct request_t { std::string tx_as_hex; bool do_not_relay; @@ -824,9 +862,10 @@ namespace cryptonote KV_SERIALIZE_OPT(do_not_relay, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::string reason; @@ -856,11 +895,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_START_MINING { - struct request + struct request_t { std::string miner_address; uint64_t threads_count; @@ -874,8 +914,9 @@ namespace cryptonote KV_SERIALIZE(ignore_battery) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -883,18 +924,20 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_GET_INFO { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; uint64_t height; @@ -966,21 +1009,23 @@ namespace cryptonote KV_SERIALIZE(version) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_STOP_MINING { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -988,27 +1033,36 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_MINING_STATUS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; bool active; uint64_t speed; uint32_t threads_count; std::string address; + std::string pow_algorithm; bool is_background_mining_enabled; + uint8_t bg_idle_threshold; + uint8_t bg_min_idle_seconds; + bool bg_ignore_battery; + uint8_t bg_target; + uint32_t block_target; + uint64_t block_reward; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) @@ -1016,23 +1070,32 @@ namespace cryptonote KV_SERIALIZE(speed) KV_SERIALIZE(threads_count) KV_SERIALIZE(address) + KV_SERIALIZE(pow_algorithm) KV_SERIALIZE(is_background_mining_enabled) + KV_SERIALIZE(bg_idle_threshold) + KV_SERIALIZE(bg_min_idle_seconds) + KV_SERIALIZE(bg_ignore_battery) + KV_SERIALIZE(bg_target) + KV_SERIALIZE(block_target) + KV_SERIALIZE(block_reward) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- struct COMMAND_RPC_SAVE_BC { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1040,6 +1103,7 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; // @@ -1047,7 +1111,7 @@ namespace cryptonote { typedef std::list<std::string> request; - struct response + struct response_t { uint64_t count; std::string status; @@ -1057,7 +1121,7 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GETBLOCKHASH @@ -1070,7 +1134,7 @@ namespace cryptonote struct COMMAND_RPC_GETBLOCKTEMPLATE { - struct request + struct request_t { uint64_t reserve_size; //max 255 bytes std::string wallet_address; @@ -1080,8 +1144,9 @@ namespace cryptonote KV_SERIALIZE(wallet_address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t difficulty; uint64_t height; @@ -1105,13 +1170,14 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SUBMITBLOCK { typedef std::vector<std::string> request; - struct response + struct response_t { std::string status; @@ -1119,11 +1185,12 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GENERATEBLOCKS { - struct request + struct request_t { uint64_t amount_of_blocks; std::string wallet_address; @@ -1133,8 +1200,9 @@ namespace cryptonote KV_SERIALIZE(wallet_address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t height; std::string status; @@ -1144,6 +1212,7 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct block_header_response @@ -1164,6 +1233,7 @@ namespace cryptonote uint64_t block_weight; uint64_t num_txes; std::string pow_hash; + uint64_t long_term_weight; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(major_version) @@ -1182,12 +1252,13 @@ namespace cryptonote KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) + KV_SERIALIZE_OPT(long_term_weight, (uint64_t)0) END_KV_SERIALIZE_MAP() }; struct COMMAND_RPC_GET_LAST_BLOCK_HEADER { - struct request + struct request_t { bool fill_pow_hash; @@ -1195,8 +1266,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1208,12 +1280,13 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH { - struct request + struct request_t { std::string hash; bool fill_pow_hash; @@ -1223,8 +1296,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1236,12 +1310,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT { - struct request + struct request_t { uint64_t height; bool fill_pow_hash; @@ -1251,8 +1325,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1264,12 +1339,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BLOCK { - struct request + struct request_t { std::string hash; uint64_t height; @@ -1281,8 +1356,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; block_header_response block_header; @@ -1302,7 +1378,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init<response_t> response; }; struct peer { @@ -1310,15 +1386,17 @@ namespace cryptonote std::string host; uint32_t ip; uint16_t port; + uint16_t rpc_port; uint64_t last_seen; + uint32_t pruning_seed; peer() = default; - peer(uint64_t id, const std::string &host, uint64_t last_seen) - : id(id), host(host), ip(0), port(0), last_seen(last_seen) + peer(uint64_t id, const std::string &host, uint64_t last_seen, uint32_t pruning_seed, uint16_t rpc_port) + : id(id), host(host), ip(0), port(0), rpc_port(rpc_port), last_seen(last_seen), pruning_seed(pruning_seed) {} - peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen) - : id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen) + peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen, uint32_t pruning_seed, uint16_t rpc_port) + : id(id), host(std::to_string(ip)), ip(ip), port(port), rpc_port(rpc_port), last_seen(last_seen), pruning_seed(pruning_seed) {} BEGIN_KV_SERIALIZE_MAP() @@ -1326,19 +1404,22 @@ namespace cryptonote KV_SERIALIZE(host) KV_SERIALIZE(ip) KV_SERIALIZE(port) + KV_SERIALIZE_OPT(rpc_port, (uint16_t)0) KV_SERIALIZE(last_seen) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; struct COMMAND_RPC_GET_PEER_LIST { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::vector<peer> white_list; @@ -1350,11 +1431,12 @@ namespace cryptonote KV_SERIALIZE(gray_list) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SET_LOG_HASH_RATE { - struct request + struct request_t { bool visible; @@ -1362,19 +1444,21 @@ namespace cryptonote KV_SERIALIZE(visible) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SET_LOG_LEVEL { - struct request + struct request_t { int8_t level; @@ -1382,19 +1466,21 @@ namespace cryptonote KV_SERIALIZE(level) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SET_LOG_CATEGORIES { - struct request + struct request_t { std::string categories; @@ -1402,8 +1488,9 @@ namespace cryptonote KV_SERIALIZE(categories) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::string categories; @@ -1413,6 +1500,7 @@ namespace cryptonote KV_SERIALIZE(categories) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct tx_info @@ -1467,13 +1555,14 @@ namespace cryptonote struct COMMAND_RPC_GET_TRANSACTION_POOL { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::vector<tx_info> transactions; @@ -1487,17 +1576,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::vector<crypto::hash> tx_hashes; @@ -1509,17 +1600,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::vector<std::string> tx_hashes; @@ -1531,6 +1624,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct tx_backlog_entry @@ -1542,13 +1636,14 @@ namespace cryptonote struct COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::vector<tx_backlog_entry> backlog; @@ -1560,6 +1655,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct txpool_histo @@ -1610,13 +1706,14 @@ namespace cryptonote struct COMMAND_RPC_GET_TRANSACTION_POOL_STATS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; txpool_stats pool_stats; @@ -1628,17 +1725,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_CONNECTIONS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::list<connection_info> connections; @@ -1648,12 +1747,12 @@ namespace cryptonote KV_SERIALIZE(connections) END_KV_SERIALIZE_MAP() }; - + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BLOCK_HEADERS_RANGE { - struct request + struct request_t { uint64_t start_height; uint64_t end_height; @@ -1665,8 +1764,9 @@ namespace cryptonote KV_SERIALIZE_OPT(fill_pow_hash, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::vector<block_header_response> headers; @@ -1678,17 +1778,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_STOP_DAEMON { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1696,17 +1798,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_FAST_EXIT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1714,17 +1818,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_LIMIT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; uint64_t limit_up; @@ -1738,11 +1844,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SET_LIMIT { - struct request + struct request_t { int64_t limit_down; // all limits (for get and set) are kB/s int64_t limit_up; @@ -1752,8 +1859,9 @@ namespace cryptonote KV_SERIALIZE(limit_up) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; int64_t limit_up; @@ -1765,19 +1873,21 @@ namespace cryptonote KV_SERIALIZE(limit_down) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_OUT_PEERS { - struct request + struct request_t { uint64_t out_peers; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(out_peers) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1785,19 +1895,21 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_IN_PEERS { - struct request + struct request_t { uint64_t in_peers; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(in_peers) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1805,17 +1917,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_START_SAVE_GRAPH { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1823,17 +1937,19 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_STOP_SAVE_GRAPH { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1841,11 +1957,12 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_HARD_FORK_INFO { - struct request + struct request_t { uint8_t version; @@ -1853,8 +1970,9 @@ namespace cryptonote KV_SERIALIZE(version) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint8_t version; bool enabled; @@ -1880,6 +1998,7 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GETBANS @@ -1897,13 +2016,14 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; std::vector<ban> bans; @@ -1913,6 +2033,7 @@ namespace cryptonote KV_SERIALIZE(bans) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SETBANS @@ -1932,7 +2053,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct request + struct request_t { std::vector<ban> bans; @@ -1940,8 +2061,9 @@ namespace cryptonote KV_SERIALIZE(bans) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1949,11 +2071,12 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_FLUSH_TRANSACTION_POOL { - struct request + struct request_t { std::vector<std::string> txids; @@ -1961,8 +2084,9 @@ namespace cryptonote KV_SERIALIZE(txids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -1970,11 +2094,12 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_OUTPUT_HISTOGRAM { - struct request + struct request_t { std::vector<uint64_t> amounts; uint64_t min_count; @@ -1990,6 +2115,7 @@ namespace cryptonote KV_SERIALIZE(recent_cutoff); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct entry { @@ -2010,7 +2136,7 @@ namespace cryptonote entry() {} }; - struct response + struct response_t { std::string status; std::vector<entry> histogram; @@ -2022,17 +2148,19 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_VERSION { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; uint32_t version; @@ -2044,11 +2172,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_COINBASE_TX_SUM { - struct request + struct request_t { uint64_t height; uint64_t count; @@ -2058,8 +2187,9 @@ namespace cryptonote KV_SERIALIZE(count); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; uint64_t emission_amount; @@ -2071,11 +2201,12 @@ namespace cryptonote KV_SERIALIZE(fee_amount) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BASE_FEE_ESTIMATE { - struct request + struct request_t { uint64_t grace_blocks; @@ -2083,8 +2214,9 @@ namespace cryptonote KV_SERIALIZE(grace_blocks) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; uint64_t fee; @@ -2098,15 +2230,17 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ALTERNATE_CHAINS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct chain_info { @@ -2127,7 +2261,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::string status; std::list<chain_info> chains; @@ -2137,11 +2271,12 @@ namespace cryptonote KV_SERIALIZE(chains) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_UPDATE { - struct request + struct request_t { std::string command; std::string path; @@ -2151,8 +2286,9 @@ namespace cryptonote KV_SERIALIZE(path); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; bool update; @@ -2172,11 +2308,12 @@ namespace cryptonote KV_SERIALIZE(path) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_RELAY_TX { - struct request + struct request_t { std::vector<std::string> txids; @@ -2184,8 +2321,9 @@ namespace cryptonote KV_SERIALIZE(txids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; @@ -2193,15 +2331,17 @@ namespace cryptonote KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SYNC_INFO { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct peer { @@ -2233,27 +2373,32 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::string status; uint64_t height; uint64_t target_height; + uint32_t next_needed_pruning_seed; std::list<peer> peers; std::list<span> spans; + std::string overview; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(height) KV_SERIALIZE(target_height) + KV_SERIALIZE(next_needed_pruning_seed) KV_SERIALIZE(peers) KV_SERIALIZE(spans) + KV_SERIALIZE(overview) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_OUTPUT_DISTRIBUTION { - struct request + struct request_t { std::vector<uint64_t> amounts; uint64_t from_height; @@ -2271,6 +2416,7 @@ namespace cryptonote KV_SERIALIZE_OPT(compress, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct distribution { @@ -2314,7 +2460,7 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::string status; std::vector<distribution> distributions; @@ -2326,11 +2472,12 @@ namespace cryptonote KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_POP_BLOCKS { - struct request + struct request_t { uint64_t nblocks; @@ -2338,8 +2485,9 @@ namespace cryptonote KV_SERIALIZE(nblocks); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string status; uint64_t height; @@ -2349,6 +2497,32 @@ namespace cryptonote KV_SERIALIZE(height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_PRUNE_BLOCKCHAIN + { + struct request_t + { + bool check; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(check, false) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + uint32_t pruning_seed; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(pruning_seed) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; }; } diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 5a754749f..b13049e61 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index e2885dbb5..14b492786 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -423,13 +423,13 @@ namespace rpc res.info.alt_blocks_count = chain.get_alternative_blocks_count(); - uint64_t total_conn = m_p2p.get_connections_count(); - res.info.outgoing_connections_count = m_p2p.get_outgoing_connections_count(); + uint64_t total_conn = m_p2p.get_public_connections_count(); + res.info.outgoing_connections_count = m_p2p.get_public_outgoing_connections_count(); res.info.incoming_connections_count = total_conn - res.info.outgoing_connections_count; - res.info.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); + res.info.white_peerlist_size = m_p2p.get_public_white_peers_count(); - res.info.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); + res.info.grey_peerlist_size = m_p2p.get_public_gray_peers_count(); res.info.mainnet = m_core.get_nettype() == MAINNET; res.info.testnet = m_core.get_nettype() == TESTNET; diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index 2c8ac3867..87161784e 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index 7c7442014..ecd3683e5 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index d2014247c..8ce56b6af 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/daemon_rpc_version.h b/src/rpc/daemon_rpc_version.h index e20af5b21..b3eb9699b 100644 --- a/src/rpc/daemon_rpc_version.h +++ b/src/rpc/daemon_rpc_version.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/instanciations.cpp b/src/rpc/instanciations.cpp index ec8882982..94fdb5f18 100644 --- a/src/rpc/instanciations.cpp +++ b/src/rpc/instanciations.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message.cpp b/src/rpc/message.cpp index 0ebe34efe..158b58005 100644 --- a/src/rpc/message.cpp +++ b/src/rpc/message.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message.h b/src/rpc/message.h index 56087b998..2b7b61ab3 100644 --- a/src/rpc/message.h +++ b/src/rpc/message.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index e09b6749e..26c5038f6 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -78,7 +78,9 @@ namespace rpc uint64_t id; uint32_t ip; uint16_t port; + uint16_t rpc_port; uint64_t last_seen; + uint32_t pruning_seed; }; struct tx_in_pool diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index 60c78480a..f2be94f51 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_args.h b/src/rpc/rpc_args.h index 8e375385b..216ba3712 100644 --- a/src/rpc/rpc_args.h +++ b/src/rpc/rpc_args.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index e0d520408..2439eaa58 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index a2ff76668..ae748e052 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/rpc/zmq_server.h b/src/rpc/zmq_server.h index 0cd906a3f..1b1e4c7cf 100644 --- a/src/rpc/zmq_server.h +++ b/src/rpc/zmq_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/CMakeLists.txt b/src/serialization/CMakeLists.txt index 5a6bebf09..28b775a37 100644 --- a/src/serialization/CMakeLists.txt +++ b/src/serialization/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2018, The Monero Project +# Copyright (c) 2016-2019, The Monero Project # # All rights reserved. # diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index f47a4494d..a0e4eff9d 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -99,7 +99,7 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> { explicit binary_archive(stream_type &s) : base_type(s) { - stream_type::streampos pos = stream_.tellg(); + stream_type::pos_type pos = stream_.tellg(); stream_.seekg(0, std::ios_base::end); eof_pos_ = stream_.tellg(); stream_.seekg(pos); diff --git a/src/serialization/binary_utils.h b/src/serialization/binary_utils.h index 79b30b337..790661e49 100644 --- a/src/serialization/binary_utils.h +++ b/src/serialization/binary_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/container.h b/src/serialization/container.h index 978a59d2a..dfe0f9691 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h index 87172834f..d2537146e 100644 --- a/src/serialization/crypto.h +++ b/src/serialization/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index c6033a399..093ca41eb 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/deque.h b/src/serialization/deque.h index 994d3f195..97f16d0a5 100644 --- a/src/serialization/deque.h +++ b/src/serialization/deque.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index 04436c21c..c9f4e175e 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 8b1af9c12..73e17a775 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // @@ -563,13 +563,13 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::connection_info& in { val.SetObject(); - auto& al = doc.GetAllocator(); INSERT_INTO_JSON_OBJECT(val, doc, incoming, info.incoming); INSERT_INTO_JSON_OBJECT(val, doc, localhost, info.localhost); INSERT_INTO_JSON_OBJECT(val, doc, local_ip, info.local_ip); INSERT_INTO_JSON_OBJECT(val, doc, ip, info.ip); INSERT_INTO_JSON_OBJECT(val, doc, port, info.port); + INSERT_INTO_JSON_OBJECT(val, doc, rpc_port, info.rpc_port); INSERT_INTO_JSON_OBJECT(val, doc, peer_id, info.peer_id); @@ -604,6 +604,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& inf GET_FROM_JSON_OBJECT(val, info.ip, ip); GET_FROM_JSON_OBJECT(val, info.port, port); + GET_FROM_JSON_OBJECT(val, info.rpc_port, rpc_port); GET_FROM_JSON_OBJECT(val, info.peer_id, peer_id); @@ -733,7 +734,9 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::peer& peer, ra INSERT_INTO_JSON_OBJECT(val, doc, id, peer.id); INSERT_INTO_JSON_OBJECT(val, doc, ip, peer.ip); INSERT_INTO_JSON_OBJECT(val, doc, port, peer.port); + INSERT_INTO_JSON_OBJECT(val, doc, rpc_port, peer.rpc_port); INSERT_INTO_JSON_OBJECT(val, doc, last_seen, peer.last_seen); + INSERT_INTO_JSON_OBJECT(val, doc, pruning_seed, peer.pruning_seed); } @@ -747,7 +750,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer) GET_FROM_JSON_OBJECT(val, peer.id, id); GET_FROM_JSON_OBJECT(val, peer.ip, ip); GET_FROM_JSON_OBJECT(val, peer.port, port); + GET_FROM_JSON_OBJECT(val, peer.rpc_port, rpc_port); GET_FROM_JSON_OBJECT(val, peer.last_seen, last_seen); + GET_FROM_JSON_OBJECT(val, peer.pruning_seed, pruning_seed); } void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx, rapidjson::Value& val) diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index b6384ca0d..c804d148b 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Monero Project +// Copyright (c) 2016-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_utils.h b/src/serialization/json_utils.h index 16f32cf66..05122f3bd 100644 --- a/src/serialization/json_utils.h +++ b/src/serialization/json_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/list.h b/src/serialization/list.h index ef0063a98..b7b65e184 100644 --- a/src/serialization/list.h +++ b/src/serialization/list.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/pair.h b/src/serialization/pair.h index 67105d530..aed0c4699 100644 --- a/src/serialization/pair.h +++ b/src/serialization/pair.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 2050e88e2..007bf265f 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/set.h b/src/serialization/set.h index edf170852..ae602a2f1 100644 --- a/src/serialization/set.h +++ b/src/serialization/set.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/string.h b/src/serialization/string.h index 97268d454..fd8450f7b 100644 --- a/src/serialization/string.h +++ b/src/serialization/string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/unordered_set.h b/src/serialization/unordered_set.h index b277f0c4a..6dd01fd93 100644 --- a/src/serialization/unordered_set.h +++ b/src/serialization/unordered_set.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/variant.h b/src/serialization/variant.h index 1d00ab461..5a179ca87 100644 --- a/src/serialization/variant.h +++ b/src/serialization/variant.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/serialization/vector.h b/src/serialization/vector.h index f180b5353..ad96582a5 100644 --- a/src/serialization/vector.h +++ b/src/serialization/vector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index e292f85dd..030867dc0 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index bdb6d2bfe..3c89c68a0 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -113,6 +113,14 @@ typedef cryptonote::simple_wallet sw; #define PRINT_USAGE(usage_help) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help; +#define LONG_PAYMENT_ID_SUPPORT_CHECK() \ + do { \ + if (!m_long_payment_id_support) { \ + fail_msg_writer() << tr("Long payment IDs are obsolete. Use --long-payment-id-support if you really must use one, and warn the recipient they are using an obsolete feature that will disappear in the future."); \ + return true; \ + } \ + } while(0) + enum TransferType { Transfer, TransferLocked, @@ -136,10 +144,12 @@ namespace const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Generate non-deterministic view and spend keys"), false}; const command_line::arg_descriptor<bool> arg_allow_mismatched_daemon_version = {"allow-mismatched-daemon-version", sw::tr("Allow communicating with a daemon that uses a different RPC version"), false}; const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; + const command_line::arg_descriptor<std::string> arg_restore_date = {"restore-date", sw::tr("Restore from estimated blockchain height on specified date"), ""}; const command_line::arg_descriptor<bool> arg_do_not_relay = {"do-not-relay", sw::tr("The newly created transaction will not be relayed to the monero network"), false}; const command_line::arg_descriptor<bool> arg_create_address_file = {"create-address-file", sw::tr("Create an address file for new wallets"), false}; const command_line::arg_descriptor<std::string> arg_subaddress_lookahead = {"subaddress-lookahead", tools::wallet2::tr("Set subaddress lookahead sizes to <major>:<minor>"), ""}; const command_line::arg_descriptor<bool> arg_use_english_language_names = {"use-english-language-names", sw::tr("Display English language names"), false}; + const command_line::arg_descriptor<bool> arg_long_payment_id_support = {"long-payment-id-support", sw::tr("Support obsolete long (unencrypted) payment ids"), false}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; @@ -150,12 +160,12 @@ namespace const char* USAGE_PAYMENTS("payments <PID_1> [<PID_2> ... <PID_N>]"); const char* USAGE_PAYMENT_ID("payment_id"); const char* USAGE_TRANSFER("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"); - const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"); - const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"); - const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id>]"); - const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id>]"); - const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id>]"); - const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id>]"); + const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id (obsolete)>]"); + const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]"); + const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]"); const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw]"); const char* USAGE_SET_LOG("set_log <level>|{+,-,}<categories>"); const char* USAGE_ACCOUNT("account\n" @@ -180,7 +190,7 @@ namespace const char* USAGE_CHECK_RESERVE_PROOF("check_reserve_proof <address> <signature_file> [<message>]"); const char* USAGE_SHOW_TRANSFERS("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); const char* USAGE_UNSPENT_OUTPUTS("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"); - const char* USAGE_RESCAN_BC("rescan_bc [hard]"); + const char* USAGE_RESCAN_BC("rescan_bc [hard|soft|keep_ki] [start_height=0]"); const char* USAGE_SET_TX_NOTE("set_tx_note <txid> [free text note]"); const char* USAGE_GET_TX_NOTE("get_tx_note <txid>"); const char* USAGE_GET_DESCRIPTION("get_description"); @@ -230,12 +240,15 @@ namespace const char* USAGE_VERSION("version"); const char* USAGE_HELP("help [<command>]"); - std::string input_line(const std::string& prompt) + std::string input_line(const std::string& prompt, bool yesno = false) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif std::cout << prompt; + if (yesno) + std::cout << " (Y/Yes/N/No)"; + std::cout << ": " << std::flush; std::string buf; #ifdef _WIN32 @@ -247,12 +260,12 @@ namespace return epee::string_tools::trim(buf); } - epee::wipeable_string input_secure_line(const std::string& prompt) + epee::wipeable_string input_secure_line(const char *prompt) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif - auto pwd_container = tools::password_container::prompt(false, prompt.c_str(), false); + auto pwd_container = tools::password_container::prompt(false, prompt, false); if (!pwd_container) { MERROR("Failed to read secure line"); @@ -425,10 +438,10 @@ namespace << ", " << dnssec_str << std::endl << sw::tr(" Monero Address = ") << addresses[0] << std::endl - << sw::tr("Is this OK? (Y/n) ") + << sw::tr("Is this OK?") ; // prompt the user for confirmation given the dns query and dnssec status - std::string confirm_dns_ok = input_line(prompt.str()); + std::string confirm_dns_ok = input_line(prompt.str(), true); if (std::cin.eof()) { return {}; @@ -540,7 +553,7 @@ namespace } catch (const tools::error::tx_rejected& e) { - fail_msg_writer() << (boost::format(sw::tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); + fail_msg_writer() << (boost::format(sw::tr("transaction %s was rejected by daemon")) % get_transaction_hash(e.tx())); std::string reason = e.reason(); if (!reason.empty()) fail_msg_writer() << sw::tr("Reason: ") << reason; @@ -596,7 +609,7 @@ namespace fail_msg_writer() << boost::format(sw::tr("File %s likely stores wallet private keys! Use a different file name.")) % filename; return false; } - return command_line::is_yes(input_line((boost::format(sw::tr("File %s already exists. Are you sure to overwrite it? (Y/Yes/N/No): ")) % filename).str())); + return command_line::is_yes(input_line((boost::format(sw::tr("File %s already exists. Are you sure to overwrite it?")) % filename).str(), true)); } return true; } @@ -860,6 +873,8 @@ bool simple_wallet::change_password(const std::vector<std::string> &args) bool simple_wallet::payment_id(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + crypto::hash payment_id; if (args.size() > 0) { @@ -1165,6 +1180,7 @@ bool simple_wallet::exchange_multisig_keys_main(const std::vector<std::string> & uint32_t threshold, total; m_wallet->multisig(NULL, &threshold, &total); success_msg_writer() << tr("Multisig wallet has been successfully created. Current wallet type: ") << threshold << "/" << total; + success_msg_writer() << tr("Multisig address: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); } } catch (const std::exception &e) @@ -2221,6 +2237,8 @@ bool simple_wallet::set_refresh_type(const std::vector<std::string> &args/* = st bool simple_wallet::set_confirm_missing_payment_id(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + const auto pwd_container = get_and_verify_password(); if (pwd_container) { @@ -2687,8 +2705,10 @@ simple_wallet::simple_wallet() " Set the wallet's refresh behaviour.\n " "priority [0|1|2|3|4]\n " " Set the fee to default/unimportant/normal/elevated/priority.\n " - "confirm-missing-payment-id <1|0>\n " + "confirm-missing-payment-id <1|0> (obsolete)\n " "ask-password <0|1|2 (or never|action|decrypt)>\n " + " action: ask the password before many actions such as transfer, etc\n " + " decrypt: same as action, but keeps the spend key encrypted in memory when not needed\n " "unit <monero|millinero|micronero|nanonero|piconero>\n " " Set the default monero (sub-)unit.\n " "min-outputs-count [n]\n " @@ -2781,7 +2801,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), tr(USAGE_RESCAN_BC), - tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); + tr("Rescan the blockchain from scratch. If \"hard\" is specified, you will lose any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), tr(USAGE_SET_TX_NOTE), @@ -2846,7 +2866,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("payment_id", boost::bind(&simple_wallet::payment_id, this, _1), tr(USAGE_PAYMENT_ID), - tr("Generate a new random full size payment id. These will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids.")); + tr("Generate a new random full size payment id (obsolete). These will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids.")); m_cmd_binder.set_handler("fee", boost::bind(&simple_wallet::print_fee_info, this, _1), tr("Print the information about the current fee and transaction backlog.")); @@ -3152,9 +3172,9 @@ bool simple_wallet::ask_wallet_create_if_needed() LOG_PRINT_L3("User asked to specify wallet file name."); wallet_path = input_line( tr(m_restoring ? "Specify a new wallet file name for your restored wallet (e.g., MyWallet).\n" - "Wallet file name (or Ctrl-C to quit): " : + "Wallet file name (or Ctrl-C to quit)" : "Specify wallet file name (e.g., MyWallet). If the wallet doesn't exist, it will be created.\n" - "Wallet file name (or Ctrl-C to quit): ") + "Wallet file name (or Ctrl-C to quit)") ); if(std::cin.eof()) { @@ -3201,7 +3221,7 @@ bool simple_wallet::ask_wallet_create_if_needed() if (!m_restoring) { message_writer() << tr("No wallet found with that name. Confirm creation of new wallet named: ") << wallet_path; - confirm_creation = input_line(tr("(Y/Yes/N/No): ")); + confirm_creation = input_line("", true); if(std::cin.eof()) { LOG_ERROR("Unexpected std::cin.eof() - Exited simple_wallet::ask_wallet_create_if_needed()"); @@ -3260,6 +3280,28 @@ static bool might_be_partial_seed(const epee::wipeable_string &words) return seed.size() < 24; } //---------------------------------------------------------------------------------------------------- +static bool datestr_to_int(const std::string &heightstr, uint16_t &year, uint8_t &month, uint8_t &day) +{ + if (heightstr.size() != 10 || heightstr[4] != '-' || heightstr[7] != '-') + { + fail_msg_writer() << tr("date format must be YYYY-MM-DD"); + return false; + } + try + { + year = boost::lexical_cast<uint16_t>(heightstr.substr(0,4)); + // lexical_cast<uint8_t> won't work because uint8_t is treated as character type + month = boost::lexical_cast<uint16_t>(heightstr.substr(5,2)); + day = boost::lexical_cast<uint16_t>(heightstr.substr(8,2)); + } + catch (const boost::bad_lexical_cast &) + { + fail_msg_writer() << tr("bad height parameter: ") << heightstr; + return false; + } + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::init(const boost::program_options::variables_map& vm) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ @@ -3388,7 +3430,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_view_key; // parse address - std::string address_string = input_line("Standard address: "); + std::string address_string = input_line("Standard address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3408,7 +3450,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) { @@ -3443,7 +3485,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_spend_key; // parse spend secret key - epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key"); if (std::cin.eof()) return false; if (spendkey_string.empty()) { @@ -3463,7 +3505,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_keys; // parse address - std::string address_string = input_line("Standard address: "); + std::string address_string = input_line("Standard address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3483,7 +3525,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse spend secret key - epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key"); if (std::cin.eof()) return false; if (spendkey_string.empty()) { @@ -3498,7 +3540,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) { @@ -3545,7 +3587,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) unsigned int multisig_n; // parse multisig type - std::string multisig_type_string = input_line("Multisig type (input as M/N with M <= N and M > 1): "); + std::string multisig_type_string = input_line("Multisig type (input as M/N with M <= N and M > 1)"); if (std::cin.eof()) return false; if (multisig_type_string.empty()) @@ -3571,7 +3613,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) message_writer() << boost::format(tr("Generating master wallet from %u of %u multisig wallet keys")) % multisig_m % multisig_n; // parse multisig address - std::string address_string = input_line("Multisig wallet address: "); + std::string address_string = input_line("Multisig wallet address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3586,7 +3628,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse secret view key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) @@ -3625,7 +3667,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) // get N secret spend keys from user for(unsigned int i=0; i<multisig_n; ++i) { - spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u):")) % (i+1) % multisig_m).str().c_str())); + spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u)")) % (i+1) % multisig_m).str().c_str())); if (std::cin.eof()) return false; if (spendkey_string.empty()) @@ -3698,15 +3740,15 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if(m_wallet->get_refresh_from_block_height() == 0) { { tools::scoped_message_writer wrt = tools::msg_writer(); - wrt << tr("No restore height is specified."); - wrt << tr("Assumed you are creating a new account, restore will be done from current estimated blockchain height."); - wrt << tr("Use --restore-height if you want to restore an already setup account from a specific height"); + wrt << tr("No restore height is specified.") << " "; + wrt << tr("Assumed you are creating a new account, restore will be done from current estimated blockchain height.") << " "; + wrt << tr("Use --restore-height or --restore-date if you want to restore an already setup account from a specific height."); } - std::string confirm = input_line(tr("Is this okay? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Is this okay?"), true); if (std::cin.eof() || !command_line::is_yes(confirm)) CHECK_AND_ASSERT_MES(false, false, tr("account creation aborted")); - m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()-1); + m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()); m_wallet->explicit_refresh_from_block_height(true); m_restore_height = m_wallet->get_refresh_from_block_height(); } @@ -3729,7 +3771,26 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_restoring && m_generate_from_json.empty() && m_generate_from_device.empty()) { - m_wallet->explicit_refresh_from_block_height(!command_line::is_arg_defaulted(vm, arg_restore_height)); + m_wallet->explicit_refresh_from_block_height(!(command_line::is_arg_defaulted(vm, arg_restore_height) || + command_line::is_arg_defaulted(vm, arg_restore_date))); + if (command_line::is_arg_defaulted(vm, arg_restore_height) && !command_line::is_arg_defaulted(vm, arg_restore_date)) + { + uint16_t year; + uint8_t month; + uint8_t day; + if (!datestr_to_int(m_restore_date, year, month, day)) + return false; + try + { + m_restore_height = m_wallet->get_blockchain_height_by_date(year, month, day); + success_msg_writer() << tr("Restore height is: ") << m_restore_height; + } + catch (const std::runtime_error& e) + { + fail_msg_writer() << e.what(); + return false; + } + } } if (!m_wallet->explicit_refresh_from_block_height() && m_restoring) { @@ -3739,9 +3800,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { std::string heightstr; if (!connected || version < MAKE_CORE_RPC_VERSION(1, 6)) - heightstr = input_line("Restore from specific blockchain height (optional, default 0): "); + heightstr = input_line("Restore from specific blockchain height (optional, default 0)"); else - heightstr = input_line("Restore from specific blockchain height (optional, default 0),\nor alternatively from specific date (YYYY-MM-DD): "); + heightstr = input_line("Restore from specific blockchain height (optional, default 0),\nor alternatively from specific date (YYYY-MM-DD)"); if (std::cin.eof()) return false; if (heightstr.empty()) @@ -3761,23 +3822,16 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("bad m_restore_height parameter: ") << heightstr; continue; } - if (heightstr.size() != 10 || heightstr[4] != '-' || heightstr[7] != '-') - { - fail_msg_writer() << tr("date format must be YYYY-MM-DD"); - continue; - } uint16_t year; uint8_t month; // 1, 2, ..., 12 uint8_t day; // 1, 2, ..., 31 try { - year = boost::lexical_cast<uint16_t>(heightstr.substr(0,4)); - // lexical_cast<uint8_t> won't work because uint8_t is treated as character type - month = boost::lexical_cast<uint16_t>(heightstr.substr(5,2)); - day = boost::lexical_cast<uint16_t>(heightstr.substr(8,2)); + if (!datestr_to_int(heightstr, year, month, day)) + return false; m_restore_height = m_wallet->get_blockchain_height_by_date(year, month, day); success_msg_writer() << tr("Restore height is: ") << m_restore_height; - std::string confirm = input_line(tr("Is this okay? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Is this okay?"), true); if (std::cin.eof()) return false; if(command_line::is_yes(confirm)) @@ -3800,7 +3854,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_restore_height >= estimate_height) { success_msg_writer() << tr("Restore height ") << m_restore_height << (" is not yet reached. The current estimated height is ") << estimate_height; - std::string confirm = input_line(tr("Still apply restore height? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Still apply restore height?"), true); if (std::cin.eof() || command_line::is_no(confirm)) m_restore_height = 0; } @@ -3861,9 +3915,11 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); m_allow_mismatched_daemon_version = command_line::get_arg(vm, arg_allow_mismatched_daemon_version); m_restore_height = command_line::get_arg(vm, arg_restore_height); + m_restore_date = command_line::get_arg(vm, arg_restore_date); m_do_not_relay = command_line::get_arg(vm, arg_do_not_relay); m_subaddress_lookahead = command_line::get_arg(vm, arg_subaddress_lookahead); m_use_english_language_names = command_line::get_arg(vm, arg_use_english_language_names); + m_long_payment_id_support = command_line::get_arg(vm, arg_long_payment_id_support); m_restoring = !m_generate_from_view_key.empty() || !m_generate_from_spend_key.empty() || !m_generate_from_keys.empty() || @@ -3873,6 +3929,14 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet || m_restore_multisig_wallet; + if (!command_line::is_arg_defaulted(vm, arg_restore_date)) + { + uint16_t year; + uint8_t month, day; + if (!datestr_to_int(m_restore_date, year, month, day)) + return false; + } + return true; } //---------------------------------------------------------------------------------------------------- @@ -3923,7 +3987,7 @@ std::string simple_wallet::get_mnemonic_language() } while (language_number < 0) { - language_choice = input_line(tr("Enter the number corresponding to the language of your choice: ")); + language_choice = input_line(tr("Enter the number corresponding to the language of your choice")); if (std::cin.eof()) return std::string(); try @@ -4562,14 +4626,10 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, tx_extra_nonce extra_nonce; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash8 payment_id8 = crypto::null_hash8; crypto::hash payment_id = crypto::null_hash; - if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) - message_writer() << - tr("NOTE: this transaction uses an encrypted payment ID: consider using subaddresses instead"); - else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) message_writer(console_color_red, false) << - tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead"); + (m_long_payment_id_support ? tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead.") : tr("WARNING: this transaction uses an unencrypted payment ID: these are obsolete. Support will be withdrawn in the future. Use subaddresses instead.")); } } if (m_auto_refresh_refreshing) @@ -4688,8 +4748,16 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo LOCK_IDLE_SCOPE(); + crypto::hash transfer_hash_pre{}; + uint64_t height_pre, height_post; + if (reset != ResetNone) - m_wallet->rescan_blockchain(reset == ResetHard, false); + { + if (reset == ResetSoftKeepKI) + height_pre = m_wallet->hash_m_transfers(-1, transfer_hash_pre); + + m_wallet->rescan_blockchain(reset == ResetHard, false, reset == ResetSoftKeepKI); + } #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4706,6 +4774,18 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money); + + if (reset == ResetSoftKeepKI) + { + m_wallet->finish_rescan_bc_keep_key_images(height_pre, transfer_hash_pre); + + height_post = m_wallet->get_num_transfer_details(); + if (height_pre != height_post) + { + message_writer() << tr("New transfer received since rescan was started. Key images are incomplete."); + } + } + ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -5268,6 +5348,8 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri bool r = true; if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); @@ -5275,19 +5357,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } - else - { - crypto::hash8 payment_id8; - if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) - { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - local_args.pop_back(); - payment_id_seen = true; - } - } - if(!r) { fail_msg_writer() << tr("payment id failed to encode"); @@ -5343,6 +5412,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri info.has_payment_id = true; } de.amount = amount; + de.original = local_args[i]; ++i; } else if (i + 1 < local_args.size()) @@ -5355,6 +5425,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); return false; } + de.original = local_args[i]; i += 2; } else @@ -5373,6 +5444,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } de.addr = info.address; de.is_subaddress = info.is_subaddress; + de.is_integrated = info.has_payment_id; num_subaddresses += info.is_subaddress; if (info.has_payment_id || !payment_id_uri.empty()) @@ -5391,6 +5463,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } @@ -5412,9 +5485,9 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return false; if (!command_line::is_yes(accepted)) @@ -5478,23 +5551,23 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<std::pair<uint64_t, uint64_t>> nblocks = m_wallet->estimate_backlog({std::make_pair(worst_fee_per_byte, worst_fee_per_byte)}); if (nblocks.size() != 1) { - prompt << "Internal error checking for backlog. " << tr("Is this okay anyway? (Y/Yes/N/No): "); + prompt << "Internal error checking for backlog. " << tr("Is this okay anyway?"); } else { if (nblocks[0].first > m_wallet->get_confirm_backlog_threshold()) - prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay? (Y/Yes/N/No): ")) % nblocks[0].first).str(); + prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay?")) % nblocks[0].first).str(); } } catch (const std::exception &e) { - prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway? (Y/Yes/N/No): "); + prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway?"); } std::string prompt_str = prompt.str(); if (!prompt_str.empty()) { - std::string accepted = input_line(prompt_str); + std::string accepted = input_line(prompt_str, true); if (std::cin.eof()) return false; if (!command_line::is_yes(accepted)) @@ -5580,9 +5653,9 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri { prompt << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); } - prompt << ENDL << tr("Is this okay? (Y/Yes/N/No): "); + prompt << ENDL << tr("Is this okay?"); - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return false; if (!command_line::is_yes(accepted)) @@ -5720,17 +5793,17 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) std::string prompt_str = tr("Sweeping ") + print_money(total_unmixable); if (ptx_vector.size() > 1) { - prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay?")) % print_money(total_unmixable) % ((unsigned long long)ptx_vector.size()) % print_money(total_fee)).str(); } else { - prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_unmixable) % print_money(total_fee)).str(); } - std::string accepted = input_line(prompt_str); + std::string accepted = input_line(prompt_str, true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -5773,7 +5846,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) catch (const tools::error::not_enough_unlocked_money& e) { fail_msg_writer() << tr("Not enough money in unlocked balance"); - std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay? (Y/Yes/N/No): ")) % print_money(e.available())).str()); + std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay?")) % print_money(e.available())).str(), true); if (std::cin.eof()) return true; if (command_line::is_yes(accepted)) @@ -5935,23 +6008,13 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); if(r) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); payment_id_seen = true; } - else - { - crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) - { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - payment_id_seen = true; - } - } if(!r && local_args.size() == 3) { @@ -5991,9 +6054,9 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6041,17 +6104,17 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st if (m_wallet->print_ring_members() && !print_ring_members(ptx_vector, prompt)) return true; if (ptx_vector.size() > 1) { - prompt << boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay?")) % print_money(total_sent) % ((unsigned long long)ptx_vector.size()) % print_money(total_fee); } else { - prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_sent) % print_money(total_fee); } - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6202,12 +6265,9 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) std::string extra_nonce; if (tools::wallet2::parse_long_payment_id(local_args.back(), payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); } - else if(tools::wallet2::parse_short_payment_id(local_args.back(), payment_id8)) - { - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - } else { fail_msg_writer() << tr("failed to parse Payment ID"); @@ -6263,9 +6323,9 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } // prompt if there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6307,10 +6367,10 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) std::ostringstream prompt; if (!print_ring_members(ptx_vector, prompt)) return true; - prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_sent) % print_money(total_fee); - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6471,14 +6531,29 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, { if (!payment_id_string.empty()) payment_id_string += ", "; - payment_id_string = std::string("encrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id8); - has_encrypted_payment_id = true; + + // if none of the addresses are integrated addresses, it's a dummy one + bool is_dummy = true; + for (const auto &e: cd.dests) + if (e.is_integrated) + is_dummy = false; + + if (is_dummy) + { + payment_id_string += std::string("dummy encrypted payment ID"); + } + else + { + payment_id_string += std::string("encrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id8); + has_encrypted_payment_id = true; + } } else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { if (!payment_id_string.empty()) payment_id_string += ", "; - payment_id_string = std::string("unencrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id); + payment_id_string += std::string("unencrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id); + payment_id_string += " (OBSOLETE)"; } } } @@ -6574,8 +6649,8 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, change_string += tr("no change"); uint64_t fee = amount - amount_to_dests; - std::string prompt_str = (boost::format(tr("Loaded %lu transactions, for %s, fee %s, %s, %s, with min ring size %lu, %s. %sIs this okay? (Y/Yes/N/No): ")) % (unsigned long)get_num_txes() % print_money(amount) % print_money(fee) % dest_string % change_string % (unsigned long)min_ring_size % payment_id_string % extra_message).str(); - return command_line::is_yes(input_line(prompt_str)); + std::string prompt_str = (boost::format(tr("Loaded %lu transactions, for %s, fee %s, %s, %s, with min ring size %lu, %s. %sIs this okay?")) % (unsigned long)get_num_txes() % print_money(amount) % print_money(fee) % dest_string % change_string % (unsigned long)min_ring_size % payment_id_string % extra_message).str(); + return command_line::is_yes(input_line(prompt_str, true)); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs) @@ -7296,6 +7371,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec const std::string type = pd.m_coinbase ? tr("block") : tr("in"); const bool unlocked = m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); transfers.push_back({ + type, pd.m_block_height, pd.m_timestamp, type, @@ -7328,6 +7404,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); transfers.push_back({ + "out", pd.m_block_height, pd.m_timestamp, "out", @@ -7365,6 +7442,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec double_spend_note = tr("[Double spend seen on the network: this transaction may or may not end up being mined] "); transfers.push_back({ "pool", + "pool", pd.m_timestamp, "in", false, @@ -7405,6 +7483,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec if ((failed && is_failed) || (!is_failed && pending)) { transfers.push_back({ (is_failed ? "failed" : "pending"), + (is_failed ? "failed" : "pending"), pd.m_timestamp, "out", false, @@ -7452,7 +7531,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) for (const auto& transfer : all_transfers) { - const auto color = transfer.confirmed ? ((transfer.direction == "in" || transfer.direction == "block") ? console_color_green : console_color_magenta) : console_color_white; + const auto color = transfer.type == "failed" ? console_color_red : transfer.confirmed ? ((transfer.direction == "in" || transfer.direction == "block") ? console_color_green : console_color_magenta) : console_color_default; std::string destinations = "-"; if (!transfer.outputs.empty()) @@ -7714,21 +7793,58 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { - bool hard = false; + uint64_t start_height = 0; + ResetType reset_type = ResetSoft; + if (!args_.empty()) { - if (args_[0] != "hard") + if (args_[0] == "hard") + { + reset_type = ResetHard; + } + else if (args_[0] == "soft") + { + reset_type = ResetSoft; + } + else if (args_[0] == "keep_ki") + { + reset_type = ResetSoftKeepKI; + } + else { PRINT_USAGE(USAGE_RESCAN_BC); return true; } - hard = true; + + if (args_.size() > 1) + { + try + { + start_height = boost::lexical_cast<uint64_t>( args_[1] ); + } + catch(const boost::bad_lexical_cast &) + { + start_height = 0; + } + } } - if (hard) + if (reset_type == ResetHard) { message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); + std::string confirm = input_line(tr("Rescan anyway?"), true); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } + } + + const uint64_t wallet_from_height = m_wallet->get_refresh_from_block_height(); + if (start_height > wallet_from_height) + { + message_writer() << tr("Warning: your restore height is higher than wallet restore height: ") << wallet_from_height; std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); if(!std::cin.eof()) { @@ -7736,7 +7852,8 @@ bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) return true; } } - return refresh_main(0, hard ? ResetHard : ResetSoft, true); + + return refresh_main(start_height, reset_type, true); } //---------------------------------------------------------------------------------------------------- void simple_wallet::check_for_messages() @@ -7770,8 +7887,9 @@ void simple_wallet::wallet_idle_thread() try { uint64_t fetched_blocks; + bool received_money; if (try_connect_to_daemon(true)) - m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks, received_money, false); // don't check the pool in background mode } catch(...) {} m_auto_refresh_refreshing = false; @@ -8203,12 +8321,13 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v { if (tools::wallet2::parse_long_payment_id(args[3], payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); description_start += 2; } else if (tools::wallet2::parse_short_payment_id(args[3], info.payment_id)) { - memcpy(payment_id.data, info.payment_id.data, 8); - description_start += 2; + fail_msg_writer() << tr("Short payment IDs are to be used within an integrated address only"); + return true; } else { @@ -8246,7 +8365,7 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v auto& row = address_book[i]; success_msg_writer() << tr("Index: ") << i; success_msg_writer() << tr("Address: ") << get_account_address_as_str(m_wallet->nettype(), row.m_is_subaddress, row.m_address); - success_msg_writer() << tr("Payment ID: ") << row.m_payment_id; + success_msg_writer() << tr("Payment ID: ") << row.m_payment_id << " (OBSOLETE)"; success_msg_writer() << tr("Description: ") << row.m_description << "\n"; } } @@ -8343,7 +8462,8 @@ bool simple_wallet::status(const std::vector<std::string> &args) { uint64_t local_height = m_wallet->get_blockchain_current_height(); uint32_t version = 0; - if (!m_wallet->check_connection(&version)) + bool ssl = false; + if (!m_wallet->check_connection(&version, &ssl)) { success_msg_writer() << "Refreshed " << local_height << "/?, no daemon connected"; return true; @@ -8355,7 +8475,7 @@ bool simple_wallet::status(const std::vector<std::string> &args) { bool synced = local_height == bc_height; success_msg_writer() << "Refreshed " << local_height << "/" << bc_height << ", " << (synced ? "synced" : "syncing") - << ", daemon RPC v" << get_version_string(version); + << ", daemon RPC v" << get_version_string(version) << ", " << (ssl ? "SSL" : "no SSL"); } else { @@ -8924,10 +9044,12 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_electrum_seed ); command_line::add_arg(desc_params, arg_allow_mismatched_daemon_version); command_line::add_arg(desc_params, arg_restore_height); + command_line::add_arg(desc_params, arg_restore_date); command_line::add_arg(desc_params, arg_do_not_relay); command_line::add_arg(desc_params, arg_create_address_file); command_line::add_arg(desc_params, arg_subaddress_lookahead); command_line::add_arg(desc_params, arg_use_english_language_names); + command_line::add_arg(desc_params, arg_long_payment_id_support); po::positional_options_description positional_options; positional_options.add(arg_command.name, -1); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index e49da8c18..cf5458175 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -84,7 +84,7 @@ namespace cryptonote std::string get_command_usage(const std::vector<std::string> &args); private: - enum ResetType { ResetNone, ResetSoft, ResetHard }; + enum ResetType { ResetNone, ResetSoft, ResetHard, ResetSoftKeepKI }; bool handle_command_line(const boost::program_options::variables_map& vm); @@ -256,6 +256,7 @@ namespace cryptonote struct transfer_view { + std::string type; boost::variant<uint64_t, std::string> block; uint64_t timestamp; std::string direction; @@ -369,6 +370,7 @@ namespace cryptonote std::string m_mnemonic_language; std::string m_import_path; std::string m_subaddress_lookahead; + std::string m_restore_date; // optional - converted to m_restore_height epee::wipeable_string m_electrum_seed; // electrum-style seed parameter @@ -396,6 +398,8 @@ namespace cryptonote bool m_auto_refresh_refreshing; std::atomic<bool> m_in_manual_refresh; uint32_t m_current_subaddress_account; + + bool m_long_payment_id_support; // MMS mms::message_store& get_message_store() const { return m_wallet->get_message_store(); }; diff --git a/src/version.cpp.in b/src/version.cpp.in index ff2405811..8aaa41b19 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -1,6 +1,6 @@ #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@" -#define DEF_MONERO_VERSION "0.13.0.0-master" -#define DEF_MONERO_RELEASE_NAME "Beryllium Bullet" +#define DEF_MONERO_VERSION "0.14.1.0" +#define DEF_MONERO_RELEASE_NAME "Boron Butterfly" #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG #include "version.h" diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index 2991f75c5..efd61cb5a 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/wallet/api/CMakeLists.txt b/src/wallet/api/CMakeLists.txt index d6f2bf6b7..3376ec70e 100644 --- a/src/wallet/api/CMakeLists.txt +++ b/src/wallet/api/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/src/wallet/api/address_book.cpp b/src/wallet/api/address_book.cpp index 7ef011e06..7be78bba7 100644 --- a/src/wallet/api/address_book.cpp +++ b/src/wallet/api/address_book.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/address_book.h b/src/wallet/api/address_book.h index f4ca68efd..92e6eaa17 100644 --- a/src/wallet/api/address_book.h +++ b/src/wallet/api/address_book.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index 913e3156f..e649f1f3a 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index 50b9f07ef..4ec7c656a 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/subaddress.cpp b/src/wallet/api/subaddress.cpp index 61dbbf4b0..8a1d34864 100644 --- a/src/wallet/api/subaddress.cpp +++ b/src/wallet/api/subaddress.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/subaddress.h b/src/wallet/api/subaddress.h index f3db7d97b..87585ec16 100644 --- a/src/wallet/api/subaddress.h +++ b/src/wallet/api/subaddress.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/subaddress_account.cpp b/src/wallet/api/subaddress_account.cpp index 19ed8fb38..9bc9d1d91 100644 --- a/src/wallet/api/subaddress_account.cpp +++ b/src/wallet/api/subaddress_account.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // @@ -62,7 +62,7 @@ void SubaddressAccountImpl::refresh() { m_rows.push_back(new SubaddressAccountRow( i, - m_wallet->m_wallet->get_subaddress_as_str({i,0}).substr(0,6), + m_wallet->m_wallet->get_subaddress_as_str({i,0}), m_wallet->m_wallet->get_subaddress_label({i,0}), cryptonote::print_money(m_wallet->m_wallet->balance(i)), cryptonote::print_money(m_wallet->m_wallet->unlocked_balance(i)) diff --git a/src/wallet/api/subaddress_account.h b/src/wallet/api/subaddress_account.h index b052182f8..358e446d4 100644 --- a/src/wallet/api/subaddress_account.h +++ b/src/wallet/api/subaddress_account.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/transaction_history.cpp b/src/wallet/api/transaction_history.cpp index ba46a6904..f4ad8b1f6 100644 --- a/src/wallet/api/transaction_history.cpp +++ b/src/wallet/api/transaction_history.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/transaction_history.h b/src/wallet/api/transaction_history.h index 7bdce97e2..67fe1989d 100644 --- a/src/wallet/api/transaction_history.h +++ b/src/wallet/api/transaction_history.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/transaction_info.cpp b/src/wallet/api/transaction_info.cpp index cc3209609..21573c6f6 100644 --- a/src/wallet/api/transaction_info.cpp +++ b/src/wallet/api/transaction_info.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/transaction_info.h b/src/wallet/api/transaction_info.h index 37e0445d9..d5c8f31cf 100644 --- a/src/wallet/api/transaction_info.h +++ b/src/wallet/api/transaction_info.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/unsigned_transaction.cpp b/src/wallet/api/unsigned_transaction.cpp index 29910a3b6..c2c04cbc3 100644 --- a/src/wallet/api/unsigned_transaction.cpp +++ b/src/wallet/api/unsigned_transaction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/unsigned_transaction.h b/src/wallet/api/unsigned_transaction.h index 8a3330014..f1af80fa1 100644 --- a/src/wallet/api/unsigned_transaction.h +++ b/src/wallet/api/unsigned_transaction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/utils.cpp b/src/wallet/api/utils.cpp index 86fe56564..24252868a 100644 --- a/src/wallet/api/utils.cpp +++ b/src/wallet/api/utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 785d2bf36..da6ddc8a3 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -942,6 +942,12 @@ uint64_t WalletImpl::approximateBlockChainHeight() const { return m_wallet->get_approximate_blockchain_height(); } + +uint64_t WalletImpl::estimateBlockChainHeight() const +{ + return m_wallet->estimate_blockchain_height(); +} + uint64_t WalletImpl::daemonBlockChainHeight() const { if(m_wallet->light_wallet()) { @@ -1401,9 +1407,11 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const if (amount) { vector<cryptonote::tx_destination_entry> dsts; cryptonote::tx_destination_entry de; + de.original = dst_addr; de.addr = info.address; de.amount = *amount; de.is_subaddress = info.is_subaddress; + de.is_integrated = info.has_payment_id; dsts.push_back(de); transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, @@ -1496,7 +1504,6 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() { clearStatus(); - vector<cryptonote::tx_destination_entry> dsts; cryptonote::tx_destination_entry de; PendingTransactionImpl * transaction = new PendingTransactionImpl(*this); @@ -1916,7 +1923,7 @@ bool WalletImpl::verifyMessageWithPublicKey(const std::string &message, const st bool WalletImpl::connectToDaemon() { - bool result = m_wallet->check_connection(NULL, DEFAULT_CONNECTION_TIMEOUT_MILLIS); + bool result = m_wallet->check_connection(NULL, NULL, DEFAULT_CONNECTION_TIMEOUT_MILLIS); if (!result) { setStatusError("Error connecting to daemon at " + m_wallet->get_daemon_address()); } else { @@ -1929,7 +1936,7 @@ bool WalletImpl::connectToDaemon() Wallet::ConnectionStatus WalletImpl::connected() const { uint32_t version = 0; - m_is_connected = m_wallet->check_connection(&version, DEFAULT_CONNECTION_TIMEOUT_MILLIS); + m_is_connected = m_wallet->check_connection(&version, NULL, DEFAULT_CONNECTION_TIMEOUT_MILLIS); if (!m_is_connected) return Wallet::ConnectionStatus_Disconnected; // Version check is not implemented in light wallets nodes/wallets diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index b4637b8e6..bd33b773c 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -109,6 +109,7 @@ public: uint64_t unlockedBalance(uint32_t accountIndex = 0) const override; uint64_t blockChainHeight() const override; uint64_t approximateBlockChainHeight() const override; + uint64_t estimateBlockChainHeight() const override; uint64_t daemonBlockChainHeight() const override; uint64_t daemonBlockChainTargetHeight() const override; bool synchronized() const override; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 82627de29..c549c260b 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -575,6 +575,12 @@ struct Wallet virtual uint64_t approximateBlockChainHeight() const = 0; /** + * @brief estimateBlockChainHeight - returns estimate blockchain height. More accurate than approximateBlockChainHeight, + * uses daemon height and falls back to calculation from date/time + * @return + **/ + virtual uint64_t estimateBlockChainHeight() const = 0; + /** * @brief daemonBlockChainHeight - returns daemon blockchain height * @return 0 - in case error communicating with the daemon. * status() will return Status_Error and errorString() will return verbose error description diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 5b262f1b7..f584e88ac 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -127,6 +127,8 @@ Wallet *WalletManagerImpl::createWalletFromDevice(const std::string &path, WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); + } else { + wallet->setRefreshFromBlockHeight(wallet->estimateBlockChainHeight()); } auto lookahead = tools::parse_subaddress_lookahead(subaddressLookahead); if (lookahead) diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index b3c0d6c00..0c83d794f 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/message_store.h b/src/wallet/message_store.h index 7d26f7889..637bd29a1 100644 --- a/src/wallet/message_store.h +++ b/src/wallet/message_store.h @@ -135,6 +135,7 @@ namespace mms { monero_address_known = false; memset(&monero_address, 0, sizeof(cryptonote::account_public_address)); + me = false; index = 0; auto_config_public_key = crypto::null_pkey; auto_config_secret_key = crypto::null_skey; diff --git a/src/wallet/message_transporter.cpp b/src/wallet/message_transporter.cpp index eafd13d3b..2f8188a3c 100644 --- a/src/wallet/message_transporter.cpp +++ b/src/wallet/message_transporter.cpp @@ -44,7 +44,7 @@ namespace mms namespace bitmessage_rpc { - struct message_info + struct message_info_t { uint32_t encodingType; std::string toAddress; @@ -66,8 +66,9 @@ namespace bitmessage_rpc KV_SERIALIZE(subject) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<message_info_t> message_info; - struct inbox_messages_response + struct inbox_messages_response_t { std::vector<message_info> inboxMessages; @@ -75,6 +76,7 @@ namespace bitmessage_rpc KV_SERIALIZE(inboxMessages) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<inbox_messages_response_t> inbox_messages_response; } @@ -116,7 +118,11 @@ bool message_transporter::receive_messages(const std::vector<std::string> &desti std::string json = get_str_between_tags(answer, "<string>", "</string>"); bitmessage_rpc::inbox_messages_response bitmessage_res; - epee::serialization::load_t_from_json(bitmessage_res, json); + if (!epee::serialization::load_t_from_json(bitmessage_res, json)) + { + MERROR("Failed to deserialize messages"); + return true; + } size_t size = bitmessage_res.inboxMessages.size(); messages.clear(); @@ -138,8 +144,10 @@ bool message_transporter::receive_messages(const std::vector<std::string> &desti std::string message_body = epee::string_encoding::base64_decode(message_info.message); // Second Base64-decoding: The MMS uses Base64 to hide non-textual data in its JSON from Bitmessage json = epee::string_encoding::base64_decode(message_body); - epee::serialization::load_t_from_json(message, json); - is_mms_message = true; + if (!epee::serialization::load_t_from_json(message, json)) + MERROR("Failed to deserialize message"); + else + is_mms_message = true; } catch(const std::exception& e) { diff --git a/src/wallet/message_transporter.h b/src/wallet/message_transporter.h index 8291311ce..736fc9b63 100644 --- a/src/wallet/message_transporter.h +++ b/src/wallet/message_transporter.h @@ -42,7 +42,7 @@ namespace mms { -struct transport_message +struct transport_message_t { cryptonote::account_public_address source_monero_address; std::string source_transport_address; @@ -78,6 +78,7 @@ struct transport_message KV_SERIALIZE(transport_id) END_KV_SERIALIZE_MAP() }; +typedef epee::misc_utils::struct_init<transport_message_t> transport_message; class message_transporter { diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 605531e59..f5f3c0e1b 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 65f13eaaa..3630aec08 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 2080c1832..0cedd4b50 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -192,6 +192,37 @@ namespace return false; } + + void add_reason(std::string &reasons, const char *reason) + { + if (!reasons.empty()) + reasons += ", "; + reasons += reason; + } + + std::string get_text_reason(const cryptonote::COMMAND_RPC_SEND_RAW_TX::response &res) + { + std::string reason; + if (res.low_mixin) + add_reason(reason, "bad ring size"); + if (res.double_spend) + add_reason(reason, "double spend"); + if (res.invalid_input) + add_reason(reason, "invalid input"); + if (res.invalid_output) + add_reason(reason, "invalid output"); + if (res.too_big) + add_reason(reason, "too big"); + if (res.overspend) + add_reason(reason, "overspend"); + if (res.fee_too_low) + add_reason(reason, "fee too low"); + if (res.not_rct) + add_reason(reason, "tx is not ringct"); + if (res.not_relayed) + add_reason(reason, "tx was not relayed"); + return reason; + } } namespace @@ -206,6 +237,12 @@ struct options { const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; const command_line::arg_descriptor<std::string> daemon_login = {"daemon-login", tools::wallet2::tr("Specify username[:password] for daemon RPC client"), "", true}; + const command_line::arg_descriptor<std::string> daemon_ssl = {"daemon-ssl", tools::wallet2::tr("Enable SSL on daemon RPC connections: enabled|disabled|autodetect"), "autodetect"}; + const command_line::arg_descriptor<std::string> daemon_ssl_private_key = {"daemon-ssl-private-key", tools::wallet2::tr("Path to a PEM format private key"), ""}; + const command_line::arg_descriptor<std::string> daemon_ssl_certificate = {"daemon-ssl-certificate", tools::wallet2::tr("Path to a PEM format certificate"), ""}; + const command_line::arg_descriptor<std::vector<std::string>> daemon_ssl_allowed_certificates = {"daemon-ssl-allowed-certificates", tools::wallet2::tr("List of paths to PEM format certificates of allowed RPC servers")}; + const command_line::arg_descriptor<std::vector<std::string>> daemon_ssl_allowed_fingerprints = {"daemon-ssl-allowed-fingerprints", tools::wallet2::tr("List of valid fingerprints of allowed RPC servers")}; + const command_line::arg_descriptor<bool> daemon_ssl_allow_any_cert = {"daemon-ssl-allow-any-cert", tools::wallet2::tr("Allow any SSL certificate from the daemon"), false}; const command_line::arg_descriptor<bool> testnet = {"testnet", tools::wallet2::tr("For testnet. Daemon must also be launched with --testnet flag"), false}; const command_line::arg_descriptor<bool> stagenet = {"stagenet", tools::wallet2::tr("For stagenet. Daemon must also be launched with --stagenet flag"), false}; const command_line::arg_descriptor<std::string, false, true, 2> shared_ringdb_dir = { @@ -277,6 +314,15 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl auto daemon_port = command_line::get_arg(vm, opts.daemon_port); auto device_name = command_line::get_arg(vm, opts.hw_device); auto device_derivation_path = command_line::get_arg(vm, opts.hw_device_derivation_path); + auto daemon_ssl_private_key = command_line::get_arg(vm, opts.daemon_ssl_private_key); + auto daemon_ssl_certificate = command_line::get_arg(vm, opts.daemon_ssl_certificate); + auto daemon_ssl_allowed_certificates = command_line::get_arg(vm, opts.daemon_ssl_allowed_certificates); + auto daemon_ssl_allowed_fingerprints = command_line::get_arg(vm, opts.daemon_ssl_allowed_fingerprints); + auto daemon_ssl_allow_any_cert = command_line::get_arg(vm, opts.daemon_ssl_allow_any_cert); + auto daemon_ssl = command_line::get_arg(vm, opts.daemon_ssl); + epee::net_utils::ssl_support_t ssl_support; + THROW_WALLET_EXCEPTION_IF(!epee::net_utils::ssl_support_from_string(ssl_support, daemon_ssl), tools::error::wallet_internal_error, + tools::wallet2::tr("Invalid argument for ") + std::string(opts.daemon_ssl.name)); THROW_WALLET_EXCEPTION_IF(!daemon_address.empty() && !daemon_host.empty() && 0 != daemon_port, tools::error::wallet_internal_error, tools::wallet2::tr("can't specify daemon host or port more than once")); @@ -327,8 +373,23 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl catch (const std::exception &e) { } } + std::list<std::string> ssl_allowed_certificates; + for (const std::string &path: daemon_ssl_allowed_certificates) + { + ssl_allowed_certificates.push_back({}); + if (!epee::file_io_utils::load_file_to_string(path, ssl_allowed_certificates.back())) + { + MERROR("Failed to load certificate: " << path); + ssl_allowed_certificates.back() = std::string(); + } + } + + std::vector<std::vector<uint8_t>> ssl_allowed_fingerprints{ daemon_ssl_allowed_fingerprints.size() }; + std::transform(daemon_ssl_allowed_fingerprints.begin(), daemon_ssl_allowed_fingerprints.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); + std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds, unattended)); - wallet->init(std::move(daemon_address), std::move(login), 0, false, *trusted_daemon); + wallet->init(std::move(daemon_address), std::move(login), 0, *trusted_daemon, ssl_support, std::make_pair(daemon_ssl_private_key, daemon_ssl_certificate), ssl_allowed_certificates, ssl_allowed_fingerprints, daemon_ssl_allow_any_cert); + boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); wallet->get_message_store().set_options(vm); @@ -342,7 +403,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl } catch (const std::exception &e) { - MERROR("Failed to parse tx notify spec"); + MERROR("Failed to parse tx notify spec: " << e.what()); } return wallet; @@ -583,19 +644,6 @@ std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_f return {nullptr, tools::password_container{}}; } -static void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) -{ - // no error - if (!status) - return; - - // empty string -> not connection - THROW_WALLET_EXCEPTION_IF(status->empty(), tools::error::no_connection_to_daemon, method); - - THROW_WALLET_EXCEPTION_IF(*status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, method); - THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, *status); -} - std::string strjoin(const std::vector<size_t> &V, const char *sep) { std::stringstream ss; @@ -610,7 +658,7 @@ std::string strjoin(const std::vector<size_t> &V, const char *sep) return ss.str(); } -static void emplace_or_replace(std::unordered_multimap<crypto::hash, tools::wallet2::pool_payment_details> &container, +static bool emplace_or_replace(std::unordered_multimap<crypto::hash, tools::wallet2::pool_payment_details> &container, const crypto::hash &key, const tools::wallet2::pool_payment_details &pd) { auto range = container.equal_range(key); @@ -619,10 +667,11 @@ static void emplace_or_replace(std::unordered_multimap<crypto::hash, tools::wall if (i->second.m_pd.m_tx_hash == pd.m_pd.m_tx_hash && i->second.m_pd.m_subaddr_index == pd.m_pd.m_subaddr_index) { i->second = pd; - return; + return false; } } container.emplace(key, pd); + return true; } void drop_from_short_history(std::list<crypto::hash> &short_chain_history, size_t N) @@ -744,9 +793,8 @@ uint64_t calculate_fee(bool use_per_byte_fee, const cryptonote::transaction &tx, return calculate_fee(base_fee, blob_size, fee_multiplier); } -crypto::hash8 get_short_payment_id(const tools::wallet2::pending_tx &ptx, hw::device &hwdev) +bool get_short_payment_id(crypto::hash8 &payment_id8, const tools::wallet2::pending_tx &ptx, hw::device &hwdev) { - crypto::hash8 payment_id8 = null_hash8; std::vector<tx_extra_field> tx_extra_fields; parse_tx_extra(ptx.tx.extra, tx_extra_fields); // ok if partially parsed cryptonote::tx_extra_nonce extra_nonce; @@ -757,19 +805,19 @@ crypto::hash8 get_short_payment_id(const tools::wallet2::pending_tx &ptx, hw::de if (ptx.dests.empty()) { MWARNING("Encrypted payment id found, but no destinations public key, cannot decrypt"); - return crypto::null_hash8; + return false; } - hwdev.decrypt_payment_id(payment_id8, ptx.dests[0].addr.m_view_public_key, ptx.tx_key); + return hwdev.decrypt_payment_id(payment_id8, ptx.dests[0].addr.m_view_public_key, ptx.tx_key); } } - return payment_id8; + return false; } tools::wallet2::tx_construction_data get_construction_data_with_decrypted_short_payment_id(const tools::wallet2::pending_tx &ptx, hw::device &hwdev) { tools::wallet2::tx_construction_data construction_data = ptx.construction_data; - crypto::hash8 payment_id = get_short_payment_id(ptx,hwdev); - if (payment_id != null_hash8) + crypto::hash8 payment_id = null_hash8; + if (get_short_payment_id(payment_id, ptx, hwdev)) { // Remove encrypted remove_field_from_tx_extra(construction_data.extra, typeid(cryptonote::tx_extra_nonce)); @@ -796,6 +844,43 @@ static void setup_shim(hw::wallet_shim * shim, tools::wallet2 * wallet) shim->get_tx_pub_key_from_received_outs = boost::bind(&tools::wallet2::get_tx_pub_key_from_received_outs, wallet, _1); } +bool get_pruned_tx(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry &entry, cryptonote::transaction &tx, crypto::hash &tx_hash) +{ + cryptonote::blobdata bd; + + // easy case if we have the whole tx + if (!entry.as_hex.empty() || (!entry.prunable_as_hex.empty() && !entry.pruned_as_hex.empty())) + { + CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.as_hex.empty() ? entry.pruned_as_hex + entry.prunable_as_hex : entry.as_hex, bd), false, "Failed to parse tx data"); + CHECK_AND_ASSERT_MES(cryptonote::parse_and_validate_tx_from_blob(bd, tx), false, "Invalid tx data"); + tx_hash = cryptonote::get_transaction_hash(tx); + // if the hash was given, check it matches + CHECK_AND_ASSERT_MES(entry.tx_hash.empty() || epee::string_tools::pod_to_hex(tx_hash) == entry.tx_hash, false, + "Response claims a different hash than the data yields"); + return true; + } + // case of a pruned tx with its prunable data hash + if (!entry.pruned_as_hex.empty() && !entry.prunable_hash.empty()) + { + crypto::hash ph; + CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.prunable_hash, ph), false, "Failed to parse prunable hash"); + CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.pruned_as_hex, bd), false, "Failed to parse pruned data"); + CHECK_AND_ASSERT_MES(parse_and_validate_tx_base_from_blob(bd, tx), false, "Invalid base tx data"); + // only v2 txes can calculate their txid after pruned + if (bd[0] > 1) + { + tx_hash = cryptonote::get_pruned_transaction_hash(tx, ph); + } + else + { + // for v1, we trust the dameon + CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.tx_hash, tx_hash), false, "Failed to parse tx hash"); + } + return true; + } + return false; +} + //----------------------------------------------------------------- } //namespace @@ -811,7 +896,7 @@ wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, const boost::optional<too w(w), locked(password != boost::none) { - if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt) + if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt || w.watch_only()) { locked = false; return; @@ -902,6 +987,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_multisig(false), m_multisig_threshold(0), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex), + m_account_public_address{crypto::null_pkey, crypto::null_pkey}, m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR), m_subaddress_lookahead_minor(SUBADDRESS_LOOKAHEAD_MINOR), m_light_wallet(false), @@ -918,6 +1004,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_last_block_reward(0), m_encrypt_keys_after_refresh(boost::none), m_unattended(unattended), + m_devices_registered(false), m_device_last_key_image_sync(0) { } @@ -957,6 +1044,12 @@ void wallet2::init_options(boost::program_options::options_description& desc_par command_line::add_arg(desc_params, opts.password_file); command_line::add_arg(desc_params, opts.daemon_port); command_line::add_arg(desc_params, opts.daemon_login); + command_line::add_arg(desc_params, opts.daemon_ssl); + command_line::add_arg(desc_params, opts.daemon_ssl_private_key); + command_line::add_arg(desc_params, opts.daemon_ssl_certificate); + command_line::add_arg(desc_params, opts.daemon_ssl_allowed_certificates); + command_line::add_arg(desc_params, opts.daemon_ssl_allowed_fingerprints); + command_line::add_arg(desc_params, opts.daemon_ssl_allow_any_cert); command_line::add_arg(desc_params, opts.testnet); command_line::add_arg(desc_params, opts.stagenet); command_line::add_arg(desc_params, opts.shared_ringdb_dir); @@ -1008,7 +1101,7 @@ std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::varia } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool ssl, bool trusted_daemon) +bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool trusted_daemon, epee::net_utils::ssl_support_t ssl_support, const std::pair<std::string, std::string> &private_key_and_certificate_path, const std::list<std::string> &allowed_certificates, const std::vector<std::vector<uint8_t>> &allowed_fingerprints, bool allow_any_cert) { m_checkpoints.init_default_checkpoints(m_nettype); if(m_http_client.is_connected()) @@ -1018,8 +1111,7 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils:: m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); m_trusted_daemon = trusted_daemon; - // When switching from light wallet to full wallet, we need to reset the height we got from lw node. - return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl); + return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl_support, private_key_and_certificate_path, allowed_certificates, allowed_fingerprints, allow_any_cert); } //---------------------------------------------------------------------------------------------------- bool wallet2::is_deterministic() const @@ -1352,6 +1444,7 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & { case rct::RCTTypeSimple: case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: return rct::decodeRctSimple(rv, rct::sk2rct(scalar1), i, mask, hwdev); case rct::RCTTypeFull: return rct::decodeRct(rv, rct::sk2rct(scalar1), i, mask, hwdev); @@ -1367,7 +1460,7 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & } } //---------------------------------------------------------------------------------------------------- -void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) +void wallet2::scan_output(const cryptonote::transaction &tx, bool miner_tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs, bool pool) { THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); @@ -1378,7 +1471,7 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi CRITICAL_REGION_LOCAL(password_lock); if (!m_encrypt_keys_after_refresh) { - boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password("output received"); + boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password(pool ? "output found in pool" : "output received"); THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); THROW_WALLET_EXCEPTION_IF(!verify_password(*pwd), error::password_needed, tr("Invalid password: password is needed to compute key image for incoming monero")); decrypt_keys(*pwd); @@ -1400,11 +1493,20 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key"); } - outs.push_back(i); - if (tx_scan_info.money_transfered == 0) + THROW_WALLET_EXCEPTION_IF(std::find(outs.begin(), outs.end(), i) != outs.end(), error::wallet_internal_error, "Same output cannot be added twice"); + if (tx_scan_info.money_transfered == 0 && !miner_tx) { tx_scan_info.money_transfered = tools::decodeRct(tx.rct_signatures, tx_scan_info.received->derivation, i, tx_scan_info.mask, m_account.get_device()); } + if (tx_scan_info.money_transfered == 0) + { + MERROR("Invalid output amount, skipping"); + tx_scan_info.error = true; + return; + } + outs.push_back(i); + THROW_WALLET_EXCEPTION_IF(tx_money_got_in_outs[tx_scan_info.received->index] >= std::numeric_limits<uint64_t>::max() - tx_scan_info.money_transfered, + error::wallet_internal_error, "Overflow in received amounts"); tx_money_got_in_outs[tx_scan_info.received->index] += tx_scan_info.money_transfered; tx_scan_info.amount = tx_scan_info.money_transfered; ++num_vouts_received; @@ -1437,7 +1539,6 @@ void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::has // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses tx_extra_additional_pub_keys additional_tx_pub_keys; - std::vector<crypto::key_derivation> additional_derivations; if (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, additional_tx_pub_keys)) { for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) @@ -1582,7 +1683,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (tx_scan_info[i].received) { hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, miner_tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1605,7 +1706,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (tx_scan_info[i].received) { hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, miner_tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1621,7 +1722,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, miner_tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1661,7 +1762,25 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; - td.m_key_image_requested = false; + if (!td.m_key_image_known) + { + // we might have cold signed, and have a mapping to key images + std::unordered_map<crypto::public_key, crypto::key_image>::const_iterator i = m_cold_key_images.find(tx_scan_info[o].in_ephemeral.pub); + if (i != m_cold_key_images.end()) + { + td.m_key_image = i->second; + td.m_key_image_known = true; + } + } + if (m_watch_only) + { + // for view wallets, that flag means "we want to request it" + td.m_key_image_request = true; + } + else + { + td.m_key_image_request = false; + } td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -1683,7 +1802,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_rct = false; } set_unspent(m_transfers.size()-1); - if (!m_multisig && !m_watch_only) + if (td.m_key_image_known) m_key_images[td.m_key_image] = m_transfers.size()-1; m_pub_keys[tx_scan_info[o].in_ephemeral.pub] = m_transfers.size()-1; if (output_tracker_cache) @@ -1939,6 +2058,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote return; } + bool all_same = true; for (const auto& i : tx_money_got_in_outs) { payment_details payment; @@ -1951,7 +2071,8 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote payment.m_coinbase = miner_tx; payment.m_subaddr_index = i.first; if (pool) { - emplace_or_replace(m_unconfirmed_payments, payment_id, pool_payment_details{payment, double_spend_seen}); + if (emplace_or_replace(m_unconfirmed_payments, payment_id, pool_payment_details{payment, double_spend_seen})) + all_same = false; if (0 != m_callback) m_callback->on_unconfirmed_money_received(height, txid, tx, payment.m_amount, payment.m_subaddr_index); } @@ -1959,13 +2080,17 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote m_payments.emplace(payment_id, payment); LOG_PRINT_L2("Payment found in " << (pool ? "pool" : "block") << ": " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount); } + + // if it's a pool tx and we already had it, don't notify again + if (pool && all_same) + notify = false; } if (notify) { std::shared_ptr<tools::Notify> tx_notify = m_tx_notify; if (tx_notify) - tx_notify->notify(epee::string_tools::pod_to_hex(txid).c_str()); + tx_notify->notify("%s", epee::string_tools::pod_to_hex(txid).c_str(), NULL); } } //---------------------------------------------------------------------------------------------------- @@ -2124,7 +2249,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblocks.bin"); THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblocks.bin"); - THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, res.status); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, get_rpc_status(res.status)); THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error, "mismatched blocks (" + boost::lexical_cast<std::string>(res.blocks.size()) + ") and output_indices (" + boost::lexical_cast<std::string>(res.output_indices.size()) + ") sizes from daemon"); @@ -2146,7 +2271,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gethashes.bin"); THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gethashes.bin"); - THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, get_rpc_status(res.status)); blocks_start_height = res.start_height; hashes = std::move(res.m_block_ids); @@ -2191,7 +2316,6 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry const cryptonote::account_keys &keys = m_account.get_keys(); auto gender = [&](wallet2::is_out_data &iod) { - boost::unique_lock<hw::device> hwdev_lock(hwdev); if (!hwdev.generate_key_derivation(iod.pkey, keys.m_view_secret_key, iod.derivation)) { MWARNING("Failed to generate key derivation from tx pubkey, skipping"); @@ -2200,12 +2324,16 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry } }; - for (auto &slot: tx_cache_data) + for (size_t i = 0; i < tx_cache_data.size(); ++i) { - for (auto &iod: slot.primary) - tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); - for (auto &iod: slot.additional) - tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + tpool.submit(&waiter, [&hwdev, &gender, &tx_cache_data, i]() { + auto &slot = tx_cache_data[i]; + boost::unique_lock<hw::device> hwdev_lock(hwdev); + for (auto &iod: slot.primary) + gender(iod); + for (auto &iod: slot.additional) + gender(iod); + }, true); } waiter.wait(&tpool); @@ -2305,11 +2433,10 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); // prepend the last 3 blocks, should be enough to guard against a block or two's reorg - std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) + auto s = std::next(prev_parsed_blocks.rbegin(), std::min((size_t)3, prev_parsed_blocks.size())).base(); + for (; s != prev_parsed_blocks.end(); ++s) { - short_chain_history.push_front(i->hash); - ++i; + short_chain_history.push_front(s->hash); } // pull the new blocks @@ -2548,7 +2675,7 @@ void wallet2::update_pool_state(bool refreshed) req.txs_hashes.push_back(epee::string_tools::pod_to_hex(p.first)); MDEBUG("asking for " << txids.size() << " transactions"); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2563,11 +2690,10 @@ void wallet2::update_pool_state(bool refreshed) { cryptonote::transaction tx; cryptonote::blobdata bd; - crypto::hash tx_hash, tx_prefix_hash; - if (epee::string_tools::parse_hexstr_to_binbuff(tx_entry.as_hex, bd)) + crypto::hash tx_hash; + + if (get_pruned_tx(tx_entry, tx, tx_hash)) { - if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash)) - { const std::vector<std::pair<crypto::hash, bool>>::const_iterator i = std::find_if(txids.begin(), txids.end(), [tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; }); if (i != txids.end()) @@ -2584,11 +2710,6 @@ void wallet2::update_pool_state(bool refreshed) { MERROR("Got txid " << tx_hash << " which we did not ask for"); } - } - else - { - LOG_PRINT_L0("failed to validate transaction from daemon"); - } } else { @@ -2608,7 +2729,7 @@ void wallet2::update_pool_state(bool refreshed) } else { - LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status); + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(res.status)); } } MTRACE("update_pool_state end"); @@ -2715,7 +2836,7 @@ std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> wallet2::create return cache; } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool) { if(m_light_wallet) { @@ -2796,13 +2917,16 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo bool first = true; while(m_run.load(std::memory_order_relaxed)) { + uint64_t next_blocks_start_height; + std::vector<cryptonote::block_complete_entry> next_blocks; + std::vector<parsed_block> next_parsed_blocks; + bool error; try { // pull the next set of blocks while we're processing the current one - uint64_t next_blocks_start_height; - std::vector<cryptonote::block_complete_entry> next_blocks; - std::vector<parsed_block> next_parsed_blocks; - bool error = false; + error = false; + next_blocks.clear(); + next_parsed_blocks.clear(); added_blocks = 0; if (!first && blocks.empty()) { @@ -2840,6 +2964,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo start_height = stop_height; throw std::runtime_error(""); // loop again } + catch (const std::exception &e) + { + MERROR("Error parsing blocks: " << e.what()); + error = true; + } blocks_fetched += added_blocks; } waiter.wait(&tpool); @@ -2882,6 +3011,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo { LOG_PRINT_L1("Another try pull_blocks (try_count=" << try_count << ")..."); first = true; + start_height = 0; + blocks.clear(); + parsed_blocks.clear(); + short_chain_history.clear(); + get_short_chain_history(short_chain_history, 1); ++try_count; } else @@ -2897,7 +3031,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo try { // If stop() is called we don't need to check pending transactions - if(m_run.load(std::memory_order_relaxed)) + if (check_pool && m_run.load(std::memory_order_relaxed)) update_pool_state(refreshed); } catch (...) @@ -3088,6 +3222,26 @@ bool wallet2::clear() m_device_last_key_image_sync = 0; return true; } +//---------------------------------------------------------------------------------------------------- +void wallet2::clear_soft(bool keep_key_images) +{ + m_blockchain.clear(); + m_transfers.clear(); + if (!keep_key_images) + m_key_images.clear(); + m_pub_keys.clear(); + m_unconfirmed_txs.clear(); + m_payments.clear(); + m_confirmed_txs.clear(); + m_unconfirmed_payments.clear(); + m_scanned_pool_txs[0].clear(); + m_scanned_pool_txs[1].clear(); + + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); +} /*! * \brief Stores wallet information to wallet file. @@ -4481,6 +4635,23 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, bool wallet2::finalize_multisig(const epee::wipeable_string &password, const std::unordered_set<crypto::public_key> &pkeys, std::vector<crypto::public_key> signers) { + bool ready; + uint32_t threshold, total; + if (!multisig(&ready, &threshold, &total)) + { + MERROR("This is not a multisig wallet"); + return false; + } + if (ready) + { + MERROR("This multisig wallet is already finalized"); + return false; + } + if (threshold + 1 != total) + { + MERROR("finalize_multisig should only be used for N-1/N wallets, use exchange_multisig_keys instead"); + return false; + } exchange_multisig_keys(password, pkeys, signers); return true; } @@ -4744,7 +4915,7 @@ bool wallet2::prepare_file_names(const std::string& file_path) return true; } //---------------------------------------------------------------------------------------------------- -bool wallet2::check_connection(uint32_t *version, uint32_t timeout) +bool wallet2::check_connection(uint32_t *version, bool *ssl, uint32_t timeout) { THROW_WALLET_EXCEPTION_IF(!m_is_initialized, error::wallet_not_initialized); @@ -4752,15 +4923,20 @@ bool wallet2::check_connection(uint32_t *version, uint32_t timeout) // TODO: Add light wallet version check. if(m_light_wallet) { - version = 0; + if (version) + *version = 0; + if (ssl) + *ssl = m_light_wallet_connected; // light wallet is always SSL return m_light_wallet_connected; } - if(!m_http_client.is_connected()) + if(!m_http_client.is_connected(ssl)) { m_node_rpc_proxy.invalidate(); if (!m_http_client.connect(std::chrono::milliseconds(timeout))) return false; + if(!m_http_client.is_connected(ssl)) + return false; } if (version) @@ -5280,7 +5456,7 @@ void wallet2::rescan_spent() m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "is_key_image_spent"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "is_key_image_spent"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != n_outputs, error::wallet_internal_error, "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(n_outputs)); @@ -5312,8 +5488,12 @@ void wallet2::rescan_spent() } } //---------------------------------------------------------------------------------------------------- -void wallet2::rescan_blockchain(bool hard, bool refresh) +void wallet2::rescan_blockchain(bool hard, bool refresh, bool keep_key_images) { + CHECK_AND_ASSERT_THROW_MES(!hard || !keep_key_images, "Cannot preserve key images on hard rescan"); + const size_t transfers_cnt = m_transfers.size(); + crypto::hash transfers_hash{}; + if(hard) { clear(); @@ -5321,25 +5501,16 @@ void wallet2::rescan_blockchain(bool hard, bool refresh) } else { - m_blockchain.clear(); - m_transfers.clear(); - m_key_images.clear(); - m_pub_keys.clear(); - m_unconfirmed_txs.clear(); - m_payments.clear(); - m_confirmed_txs.clear(); - m_unconfirmed_payments.clear(); - m_scanned_pool_txs[0].clear(); - m_scanned_pool_txs[1].clear(); - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); + if (keep_key_images && refresh) + hash_m_transfers((int64_t) transfers_cnt, transfers_hash); + clear_soft(keep_key_images); } if (refresh) this->refresh(false); + + if (refresh && keep_key_images) + finish_rescan_bc_keep_key_images(transfers_cnt, transfers_hash); } //---------------------------------------------------------------------------------------------------- bool wallet2::is_transfer_unlocked(const transfer_details& td) const @@ -5613,7 +5784,7 @@ void wallet2::commit_tx(pending_tx& ptx) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "submit_raw_tx"); // MyMonero and OpenMonero use different status strings - THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, ores.status, ores.error); + THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, get_rpc_status(ores.status), ores.error); } else { @@ -5627,7 +5798,7 @@ void wallet2::commit_tx(pending_tx& ptx) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "sendrawtransaction"); THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "sendrawtransaction"); - THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, daemon_send_resp.status, daemon_send_resp.reason); + THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp)); // sanity checks for (size_t idx: ptx.selected_transfers) { @@ -5824,15 +5995,16 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; if (sd.use_bulletproofs) { - range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; } crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, range_proof_type, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, rct_config, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -5877,6 +6049,61 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin txs.back().additional_tx_keys = additional_tx_keys; } + // add key image mapping for these txes + const account_keys &keys = get_account().get_keys(); + hw::device &hwdev = m_account.get_device(); + for (size_t n = 0; n < exported_txs.txes.size(); ++n) + { + const cryptonote::transaction &tx = signed_txes.ptx[n].tx; + + crypto::key_derivation derivation; + std::vector<crypto::key_derivation> additional_derivations; + + // compute public keys from out secret keys + crypto::public_key tx_pub_key; + crypto::secret_key_to_public_key(txs[n].tx_key, tx_pub_key); + std::vector<crypto::public_key> additional_tx_pub_keys; + for (const crypto::secret_key &skey: txs[n].additional_tx_keys) + { + additional_tx_pub_keys.resize(additional_tx_pub_keys.size() + 1); + crypto::secret_key_to_public_key(skey, additional_tx_pub_keys.back()); + } + + // compute derivations + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey in " << cryptonote::get_transaction_hash(tx) << ", skipping"); + static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); + } + for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) + { + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from additional tx pubkey in " << cryptonote::get_transaction_hash(tx) << ", skipping"); + memcpy(&additional_derivations.back(), rct::identity().bytes, sizeof(crypto::key_derivation)); + } + } + + for (size_t i = 0; i < tx.vout.size(); ++i) + { + if (tx.vout[i].target.type() != typeid(cryptonote::txout_to_key)) + continue; + const cryptonote::txout_to_key &out = boost::get<cryptonote::txout_to_key>(tx.vout[i].target); + // if this output is back to this wallet, we can calculate its key image already + if (!is_out_to_acc_precomp(m_subaddresses, out.key, derivation, additional_derivations, i, hwdev)) + continue; + crypto::key_image ki; + cryptonote::keypair in_ephemeral; + if (generate_key_image_helper(keys, m_subaddresses, out.key, tx_pub_key, additional_tx_pub_keys, i, in_ephemeral, ki, hwdev)) + signed_txes.tx_key_images[out.key] = ki; + else + MERROR("Failed to calculate key image"); + } + } + // add key images signed_txes.key_images.resize(m_transfers.size()); for (size_t i = 0; i < m_transfers.size(); ++i) @@ -6039,6 +6266,10 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too bool r = import_key_images(signed_txs.key_images); if (!r) return false; + // remember key images for this tx, for when we get those txes from the blockchain + for (const auto &e: signed_txs.tx_key_images) + m_cold_key_images.insert(e); + ptx = signed_txs.ptx; return true; @@ -6118,17 +6349,17 @@ bool wallet2::save_multisig_tx(const std::vector<pending_tx>& ptx_vector, const return epee::file_io_utils::save_string_to_file(filename, ciphertext); } //---------------------------------------------------------------------------------------------------- -bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func) +bool wallet2::parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx_set &exported_txs) const { const size_t magiclen = strlen(MULTISIG_UNSIGNED_TX_PREFIX); - if (strncmp(s.c_str(), MULTISIG_UNSIGNED_TX_PREFIX, magiclen)) + if (strncmp(multisig_tx_st.c_str(), MULTISIG_UNSIGNED_TX_PREFIX, magiclen)) { LOG_PRINT_L0("Bad magic from multisig tx data"); return false; } try { - s = decrypt_with_view_secret_key(std::string(s, magiclen)); + multisig_tx_st = decrypt_with_view_secret_key(std::string(multisig_tx_st, magiclen)); } catch (const std::exception &e) { @@ -6137,7 +6368,7 @@ bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported } try { - std::istringstream iss(s); + std::istringstream iss(multisig_tx_st); boost::archive::portable_binary_iarchive ar(iss); ar >> exported_txs; } @@ -6159,6 +6390,17 @@ bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported CHECK_AND_ASSERT_MES(ptx.construction_data.sources.size() == ptx.tx.vin.size(), false, "Mismatched sources/vin sizes"); } + return true; +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func) +{ + if(!parse_multisig_tx_from_str(s, exported_txs)) + { + LOG_PRINT_L0("Failed to parse multisig transaction from string"); + return false; + } + LOG_PRINT_L1("Loaded multisig tx unsigned data from binary: " << exported_txs.m_ptx.size() << " transactions"); for (auto &ptx: exported_txs.m_ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(ptx.tx)); @@ -6235,12 +6477,13 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; if (sd.use_bulletproofs) { - range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; } - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, rct_config, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -6518,7 +6761,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblockheadersrange"); THROW_WALLET_EXCEPTION_IF(getbh_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblockheadersrange"); - THROW_WALLET_EXCEPTION_IF(getbh_res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, getbh_res.status); + THROW_WALLET_EXCEPTION_IF(getbh_res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, get_rpc_status(getbh_res.status)); if (getbh_res.headers.size() != N) { MERROR("Bad blockheaders size"); @@ -6690,11 +6933,12 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); // get those transactions from the daemon + auto it = txs_hashes.begin(); static const size_t SLICE_SIZE = 200; for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) { req.decode_as_json = false; - req.prune = false; + req.prune = true; req.txs_hashes.clear(); size_t ntxes = slice + SLICE_SIZE > txs_hashes.size() ? txs_hashes.size() - slice : SLICE_SIZE; for (size_t s = slice; s < slice + ntxes; ++s) @@ -6713,19 +6957,15 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Scanning " << res.txs.size() << " transactions"); THROW_WALLET_EXCEPTION_IF(slice + res.txs.size() > txs_hashes.size(), error::wallet_internal_error, "Unexpected tx array size"); - auto it = req.txs_hashes.begin(); for (size_t i = 0; i < res.txs.size(); ++i, ++it) { const auto &tx_info = res.txs[i]; - THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != epee::string_tools::pod_to_hex(txs_hashes[slice + i]), error::wallet_internal_error, "Wrong txid received"); - THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); - cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); - THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); + cryptonote::transaction tx; + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(tx_info, tx, tx_hash), error::wallet_internal_error, + "Failed to get transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!(tx_hash == *it), error::wallet_internal_error, "Wrong txid received"); + THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); } } @@ -6980,7 +7220,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, get_rpc_status(resp_t.status)); } // if we want to segregate fake outs pre or post fork, get distribution @@ -7003,7 +7243,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_distribution"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, resp_t.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, get_rpc_status(resp_t.status)); // check we got all data for(size_t idx: selected_transfers) @@ -7402,7 +7642,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); @@ -7655,7 +7895,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, {}, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -7704,7 +7944,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config) { using namespace cryptonote; // throw if attempting a transaction with no destinations @@ -7886,7 +8126,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); auto sources_copy = sources; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, range_proof_type, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, rct_config, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -7931,7 +8171,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry LOG_PRINT_L2("Creating supplementary multisig transaction"); cryptonote::transaction ms_tx; auto sources_copy_copy = sources_copy; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, range_proof_type, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, rct_config, &msout, false); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -8237,7 +8477,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; - td.m_key_image_requested = false; + td.m_key_image_request = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -8608,15 +8848,16 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp TX() : weight(0), needed_fee(0) {} - void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { + void add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) { std::vector<cryptonote::tx_destination_entry>::iterator i; - i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &addr, sizeof(addr)); }); + i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &de.addr, sizeof(de.addr)); }); if (i == dsts.end()) { - dsts.push_back(tx_destination_entry(0,addr,is_subaddress)); + dsts.push_back(de); i = dsts.end() - 1; + i->amount = 0; } i->amount += amount; } @@ -8625,8 +8866,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp THROW_WALLET_EXCEPTION_IF(original_output_index > dsts.size(), error::wallet_internal_error, std::string("original_output_index too large: ") + std::to_string(original_output_index) + " > " + std::to_string(dsts.size())); if (original_output_index == dsts.size()) - dsts.push_back(tx_destination_entry(0,addr,is_subaddress)); - THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &addr, sizeof(addr)), error::wallet_internal_error, "Mismatched destination address"); + { + dsts.push_back(de); + dsts.back().amount = 0; + } + THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &de.addr, sizeof(de.addr)), error::wallet_internal_error, "Mismatched destination address"); dsts[original_output_index].amount += amount; } } @@ -8638,7 +8882,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); const bool use_rct = use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const rct::RCTConfig rct_config { + bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean, + bulletproof ? (use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1) : 0 + }; const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); @@ -8894,7 +9141,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(dsts[0].amount)); - tx.add(dsts[0].addr, dsts[0].is_subaddress, dsts[0].amount, original_output_index, m_merge_destinations); + tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations); available_amount -= dsts[0].amount; dsts[0].amount = 0; pop_index(dsts, 0); @@ -8905,7 +9152,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); - tx.add(dsts[0].addr, dsts[0].is_subaddress, available_amount, original_output_index, m_merge_destinations); + tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations); dsts[0].amount -= available_amount; available_amount = 0; } @@ -8927,6 +9174,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp { const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); try_tx = dsts.empty() || (estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); + THROW_WALLET_EXCEPTION_IF(try_tx && tx.dsts.empty(), error::tx_too_big, estimated_rct_tx_weight, upper_transaction_weight_limit); } } @@ -8951,7 +9199,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.selected_transfers.size() << " inputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -8994,7 +9242,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while (needed_fee > test_ptx.fee) { if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9067,7 +9315,7 @@ skip_tx: extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ - range_proof_type); + rct_config); } else { transfer_selected(tx.dsts, tx.selected_transfers, @@ -9207,7 +9455,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const rct::RCTConfig rct_config { + bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean, + bulletproof ? (use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1) : 0, + }; const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); const uint64_t fee_quantization_mask = get_fee_quantization_mask(); @@ -9283,7 +9534,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.selected_transfers.size() << " outputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9320,7 +9571,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton } if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9359,7 +9610,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton pending_tx test_ptx; if (use_rct) { transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); } else { transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9463,7 +9714,7 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); throw_on_rpc_response_error(result, "get_hard_fork_info"); - bool close_enough = height >= earliest_height - early_blocks; // start using the rules that many blocks beforehand + bool close_enough = height >= earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand if (close_enough) LOG_PRINT_L2("Using v" << (unsigned)version << " rules"); else @@ -9654,7 +9905,7 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9667,11 +9918,10 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, + "Failed to get transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); std::vector<tx_extra_field> tx_extra_fields; THROW_WALLET_EXCEPTION_IF(!parse_tx_extra(tx.extra, tx_extra_fields), error::wallet_internal_error, "Transaction extra has unsupported format"); @@ -9705,7 +9955,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9718,12 +9968,10 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "Failed to get tx from daemon"); std::vector<std::vector<crypto::signature>> signatures; @@ -9825,7 +10073,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9838,12 +10086,10 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "failed to get tx from daemon"); // check signature size size_t num_sigs = 0; @@ -9950,24 +10196,30 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, @@ -10006,7 +10258,7 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de crypto::secret_key scalar1; hwdev.derivation_to_scalar(found_derivation, n, scalar1); rct::ecdhTuple ecdh_info = tx.rct_signatures.ecdhInfo[n]; - hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1)); + hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1), tx.rct_signatures.type == rct::RCTTypeBulletproof2); const rct::key C = tx.rct_signatures.outPk[n].mask; rct::key Ctmp; THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.mask.bytes) != 0, error::wallet_internal_error, "Bad ECDH input mask"); @@ -10090,24 +10342,30 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); @@ -10202,24 +10460,30 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); @@ -10438,7 +10702,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr for (size_t i = 0; i < proofs.size(); ++i) gettx_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(proofs[i].txid)); gettx_req.decode_as_json = false; - gettx_req.prune = false; + gettx_req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -10462,14 +10726,11 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const reserve_proof_entry& proof = proofs[i]; THROW_WALLET_EXCEPTION_IF(gettx_res.txs[i].in_pool, error::wallet_internal_error, "Tx is unconfirmed"); - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(gettx_res.txs[i].as_hex, tx_data); + cryptonote::transaction tx; + crypto::hash tx_hash; + ok = get_pruned_tx(gettx_res.txs[i], tx, tx_hash); THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != proof.txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); THROW_WALLET_EXCEPTION_IF(proof.index_in_tx >= tx.vout.size(), error::wallet_internal_error, "index_in_tx is out of bound"); @@ -10511,7 +10772,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr crypto::secret_key shared_secret; crypto::derivation_to_scalar(derivation, proof.index_in_tx, shared_secret); rct::ecdhTuple ecdh_info = tx.rct_signatures.ecdhInfo[proof.index_in_tx]; - rct::ecdhDecode(ecdh_info, rct::sk2rct(shared_secret)); + rct::ecdhDecode(ecdh_info, rct::sk2rct(shared_secret), tx.rct_signatures.type == rct::RCTTypeBulletproof2); amount = rct::h2d(ecdh_info.amount); } total += amount; @@ -10550,7 +10811,10 @@ uint64_t wallet2::get_daemon_blockchain_height(string &err) const boost::optional<std::string> result = m_node_rpc_proxy.get_height(height); if (result) { - err = *result; + if (m_trusted_daemon) + err = *result; + else + err = "daemon error"; return 0; } @@ -10565,7 +10829,10 @@ uint64_t wallet2::get_daemon_blockchain_target_height(string &err) const auto result = m_node_rpc_proxy.get_target_height(target_height); if (result && *result != CORE_RPC_STATUS_OK) { - err= *result; + if (m_trusted_daemon) + err = *result; + else + err = "daemon error"; return 0; } return target_height; @@ -10833,23 +11100,23 @@ bool wallet2::export_key_images(const std::string &filename) const } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images() const +std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const { PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; size_t offset = 0; - while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_requested) - ++offset; + if (!all) + { + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_request) + ++offset; + } ski.reserve(m_transfers.size() - offset); for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; - crypto::hash hash; - crypto::cn_fast_hash(&td.m_key_image, sizeof(td.m_key_image), hash); - // get ephemeral public key const cryptonote::tx_out &out = td.m_tx.vout[td.m_internal_output_index]; THROW_WALLET_EXCEPTION_IF(out.target.type() != typeid(txout_to_key), error::wallet_internal_error, @@ -11001,7 +11268,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag m_transfers[n + offset].m_key_image = signed_key_images[n].first; m_key_images[m_transfers[n + offset].m_key_image] = n + offset; m_transfers[n + offset].m_key_image_known = true; - m_transfers[n + offset].m_key_image_requested = false; + m_transfers[n + offset].m_key_image_request = false; m_transfers[n + offset].m_key_image_partial = false; } PERF_TIMER_STOP(import_key_images_B); @@ -11073,7 +11340,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag COMMAND_RPC_GET_TRANSACTIONS::request gettxs_req; COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res; gettxs_req.decode_as_json = false; - gettxs_req.prune = false; + gettxs_req.prune = true; gettxs_req.txs_hashes.reserve(spent_txids.size()); for (const crypto::hash& spent_txid : spent_txids) gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid)); @@ -11093,17 +11360,16 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag PERF_TIMER_START(import_key_images_F); auto spent_txid = spent_txids.begin(); hw::device &hwdev = m_account.get_device(); + auto it = spent_txids.begin(); for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs) { THROW_WALLET_EXCEPTION_IF(e.in_pool, error::wallet_internal_error, "spent tx isn't supposed to be in txpool"); - // parse tx - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(e.as_hex, bd), error::wallet_internal_error, "parse_hexstr_to_binbuff failed"); cryptonote::transaction spent_tx; - crypto::hash spnet_txid_parsed, spent_txid_prefix; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, spent_tx, spnet_txid_parsed, spent_txid_prefix), error::wallet_internal_error, "parse_and_validate_tx_from_blob failed"); - THROW_WALLET_EXCEPTION_IF(*spent_txid != spnet_txid_parsed, error::wallet_internal_error, "parsed txid mismatch"); + crypto::hash spnet_txid_parsed; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(e, spent_tx, spnet_txid_parsed), error::wallet_internal_error, "Failed to get tx from daemon"); + THROW_WALLET_EXCEPTION_IF(!(spnet_txid_parsed == *it), error::wallet_internal_error, "parsed txid mismatch"); + ++it; // get received (change) amount uint64_t tx_money_got_in_outs = 0; @@ -11121,6 +11387,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key derivation"); } size_t output_index = 0; + bool miner_tx = cryptonote::is_coinbase(spent_tx); for (const cryptonote::tx_out& out : spent_tx.vout) { tx_scan_info_t tx_scan_info; @@ -11128,11 +11395,13 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag THROW_WALLET_EXCEPTION_IF(tx_scan_info.error, error::wallet_internal_error, "check_acc_out_precomp failed"); if (tx_scan_info.received) { - if (tx_scan_info.money_transfered == 0) + if (tx_scan_info.money_transfered == 0 && !miner_tx) { rct::key mask; tx_scan_info.money_transfered = tools::decodeRct(spent_tx.rct_signatures, tx_scan_info.received->derivation, output_index, mask, hwdev); } + THROW_WALLET_EXCEPTION_IF(tx_money_got_in_outs >= std::numeric_limits<uint64_t>::max() - tx_scan_info.money_transfered, + error::wallet_internal_error, "Overflow in received amounts"); tx_money_got_in_outs += tx_scan_info.money_transfered; } ++output_index; @@ -11149,6 +11418,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag auto it = m_key_images.find(boost::get<cryptonote::txin_to_key>(in).k_image); if (it != m_key_images.end()) { + THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), error::wallet_internal_error, std::string("Key images cache contains illegal transfer offset: ") + std::to_string(it->second) + std::string(" m_transfers.size() = ") + std::to_string(m_transfers.size())); const transfer_details& td = m_transfers[it->second]; uint64_t amount = boost::get<cryptonote::txin_to_key>(in).amount; if (amount > 0) @@ -11220,7 +11490,7 @@ bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) td.m_key_image = key_images[i]; m_key_images[m_transfers[i].m_key_image] = i; td.m_key_image_known = true; - td.m_key_image_requested = false; + td.m_key_image_request = false; td.m_key_image_partial = false; m_pub_keys[m_transfers[i].get_public_key()] = i; } @@ -11292,7 +11562,7 @@ std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export std::vector<tools::wallet2::transfer_details> outs; size_t offset = 0; - while (offset < m_transfers.size() && m_transfers[offset].m_key_image_known) + while (offset < m_transfers.size() && (m_transfers[offset].m_key_image_known && !m_transfers[offset].m_key_image_request)) ++offset; outs.reserve(m_transfers.size() - offset); @@ -11336,7 +11606,7 @@ size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet const size_t original_size = m_transfers.size(); m_transfers.resize(offset + outputs.second.size()); for (size_t i = 0; i < offset; ++i) - m_transfers[i].m_key_image_requested = false; + m_transfers[i].m_key_image_request = false; for (size_t i = 0; i < outputs.second.size(); ++i) { transfer_details td = outputs.second[i]; @@ -11377,7 +11647,7 @@ process: THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; - td.m_key_image_requested = true; + td.m_key_image_request = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i + offset)); @@ -11623,7 +11893,7 @@ void wallet2::update_multisig_rescan_info(const std::vector<std::vector<rct::key m_key_images.erase(td.m_key_image); td.m_key_image = get_multisig_composite_key_image(n); td.m_key_image_known = true; - td.m_key_image_requested = false; + td.m_key_image_request = false; td.m_key_image_partial = false; td.m_multisig_k = multisig_k[n]; m_key_images[td.m_key_image] = n; @@ -12006,7 +12276,7 @@ uint64_t wallet2::get_blockchain_height_by_date(uint16_t year, uint8_t month, ui else if (res.status == CORE_RPC_STATUS_BUSY) oss << "daemon is busy"; else - oss << res.status; + oss << get_rpc_status(res.status); throw std::runtime_error(oss.str()); } cryptonote::block blk_min, blk_mid, blk_max; @@ -12225,4 +12495,83 @@ void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & pass if (0 != m_callback) m_callback->on_passphrase_request(on_device, passphrase); } +//---------------------------------------------------------------------------------------------------- +std::string wallet2::get_rpc_status(const std::string &s) const +{ + if (m_trusted_daemon) + return s; + return "<error>"; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const +{ + // no error + if (!status) + return; + + MERROR("RPC error: " << method << ": status " << *status); + + // empty string -> not connection + THROW_WALLET_EXCEPTION_IF(status->empty(), tools::error::no_connection_to_daemon, method); + + THROW_WALLET_EXCEPTION_IF(*status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, method); + THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, m_trusted_daemon ? *status : "daemon error"); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::hash_m_transfer(const transfer_details & transfer, crypto::hash &hash) const +{ + KECCAK_CTX state; + keccak_init(&state); + keccak_update(&state, (const uint8_t *) transfer.m_txid.data, sizeof(transfer.m_txid.data)); + keccak_update(&state, (const uint8_t *) transfer.m_internal_output_index, sizeof(transfer.m_internal_output_index)); + keccak_update(&state, (const uint8_t *) transfer.m_global_output_index, sizeof(transfer.m_global_output_index)); + keccak_update(&state, (const uint8_t *) transfer.m_amount, sizeof(transfer.m_amount)); + keccak_finish(&state, (uint8_t *) hash.data); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const +{ + CHECK_AND_ASSERT_THROW_MES(transfer_height > (int64_t)m_transfers.size(), "Hash height is greater than number of transfers"); + + KECCAK_CTX state; + crypto::hash tmp_hash{}; + uint64_t current_height = 0; + + keccak_init(&state); + for(const transfer_details & transfer : m_transfers){ + if (transfer_height >= 0 && current_height >= (uint64_t)transfer_height){ + break; + } + + hash_m_transfer(transfer, tmp_hash); + keccak_update(&state, (const uint8_t *) transfer.m_block_height, sizeof(transfer.m_block_height)); + keccak_update(&state, (const uint8_t *) tmp_hash.data, sizeof(tmp_hash.data)); + current_height += 1; + } + + keccak_finish(&state, (uint8_t *) hash.data); + return current_height; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash) +{ + // Compute hash of m_transfers, if differs there had to be BC reorg. + crypto::hash new_transfers_hash{}; + hash_m_transfers((int64_t) transfer_height, new_transfers_hash); + + if (new_transfers_hash != hash) + { + // Soft-Reset to avoid inconsistency in case of BC reorg. + clear_soft(false); // keep_key_images works only with soft reset. + THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, "Transfers changed during rescan, soft or hard rescan is needed"); + } + + // Restore key images in m_transfers from m_key_images + for(auto it = m_key_images.begin(); it != m_key_images.end(); it++) + { + THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), error::wallet_internal_error, "Key images cache contains illegal transfer offset"); + m_transfers[it->second].m_key_image = it->first; + m_transfers[it->second].m_key_image_known = true; + } +} } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 5b1988080..1911a8e96 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -67,6 +67,7 @@ #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2" class Serialization_portability_wallet_Test; +class wallet_accessor_test; namespace tools { @@ -171,6 +172,7 @@ namespace tools class wallet2 { friend class ::Serialization_portability_wallet_Test; + friend class ::wallet_accessor_test; friend class wallet_keys_unlocker; friend class wallet_device_callback; public: @@ -267,7 +269,7 @@ namespace tools uint64_t m_amount; bool m_rct; bool m_key_image_known; - bool m_key_image_requested; + bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested size_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; @@ -292,7 +294,7 @@ namespace tools FIELD(m_amount) FIELD(m_rct) FIELD(m_key_image_known) - FIELD(m_key_image_requested) + FIELD(m_key_image_request) FIELD(m_pk_index) FIELD(m_subaddr_index) FIELD(m_key_image_partial) @@ -448,6 +450,7 @@ namespace tools { std::vector<pending_tx> ptx; std::vector<crypto::key_image> key_images; + std::unordered_map<crypto::public_key, crypto::key_image> tx_key_images; }; struct multisig_tx_set @@ -675,7 +678,12 @@ namespace tools bool deinit(); bool init(std::string daemon_address = "http://localhost:8080", - boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, bool ssl = false, bool trusted_daemon = false); + boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, + bool trusted_daemon = true, + epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect, + const std::pair<std::string, std::string> &private_key_and_certificate_path = {}, + const std::list<std::string> &allowed_certificates = {}, const std::vector<std::vector<uint8_t>> &allowed_fingerprints = {}, + bool allow_any_cert = false); void stop() { m_run.store(false, std::memory_order_relaxed); m_message_store.stop(); } @@ -732,7 +740,7 @@ namespace tools bool is_deprecated() const; void refresh(bool trusted_daemon); void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched); - void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true); bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok); void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; } @@ -763,7 +771,7 @@ namespace tools uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); void transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type); + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config); void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); @@ -792,6 +800,7 @@ namespace tools void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); + bool parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx_set &exported_txs) const; bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); @@ -799,7 +808,7 @@ namespace tools bool sign_multisig_tx_to_file(multisig_tx_set &exported_txs, const std::string &filename, std::vector<crypto::hash> &txids); std::vector<pending_tx> create_unmixable_sweep_transactions(); void discard_unmixable_outputs(); - bool check_connection(uint32_t *version = NULL, uint32_t timeout = 200000); + bool check_connection(uint32_t *version = NULL, bool *ssl = NULL, uint32_t timeout = 200000); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; void get_payments(std::list<std::pair<crypto::hash,wallet2::payment_details>>& payments, uint64_t min_height, uint64_t max_height = (uint64_t)-1, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; @@ -810,7 +819,7 @@ namespace tools uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); - void rescan_blockchain(bool hard, bool refresh = true); + void rescan_blockchain(bool hard, bool refresh = true, bool keep_key_images = false); bool is_transfer_unlocked(const transfer_details& td) const; bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; @@ -927,6 +936,9 @@ namespace tools if(ver < 27) return; a & m_device_last_key_image_sync; + if(ver < 28) + return; + a & m_cold_key_images; } /*! @@ -1113,7 +1125,7 @@ namespace tools std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename) const; - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images() const; + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector<crypto::key_image> key_images); @@ -1237,6 +1249,9 @@ namespace tools void set_tx_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_tx_notify = notify; } bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; + void hash_m_transfer(const transfer_details & transfer, crypto::hash &hash) const; + uint64_t hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const; + void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash); private: /*! @@ -1258,6 +1273,7 @@ namespace tools void detach_blockchain(uint64_t height); void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; bool clear(); + void clear_soft(bool keep_key_images=false); void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); @@ -1288,7 +1304,7 @@ namespace tools bool tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; std::vector<size_t> get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const; - void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); + void scan_output(const cryptonote::transaction &tx, bool miner_tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs, bool pool); void trim_hashchain(); crypto::key_image get_multisig_composite_key_image(size_t n) const; rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const std::unordered_set<crypto::public_key> &ignore_set, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; @@ -1326,6 +1342,9 @@ namespace tools void on_pin_request(epee::wipeable_string & pin); void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + std::string get_rpc_status(const std::string &s) const; + void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const; + cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; std::string m_daemon_address; @@ -1355,6 +1374,7 @@ namespace tools uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; + std::unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images; std::atomic<bool> m_run; @@ -1449,7 +1469,7 @@ namespace tools std::unique_ptr<wallet_device_callback> m_device_callback; }; } -BOOST_CLASS_VERSION(tools::wallet2, 27) +BOOST_CLASS_VERSION(tools::wallet2, 28) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 11) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) @@ -1461,7 +1481,7 @@ BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) -BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 0) +BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1) BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1510,7 +1530,7 @@ namespace boost } if (ver < 10) { - x.m_key_image_requested = false; + x.m_key_image_request = false; } } @@ -1598,7 +1618,7 @@ namespace boost initialize_transfer_details(a, x, ver); return; } - a & x.m_key_image_requested; + a & x.m_key_image_request; if (ver < 11) return; a & x.m_uses; @@ -1798,6 +1818,9 @@ namespace boost { a & x.ptx; a & x.key_images; + if (ver < 1) + return; + a & x.tx_key_images; } template <class Archive> diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index b9d0a6a75..a4bb342ca 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/wallet_args.h b/src/wallet/wallet_args.h index a1f251144..c861dca11 100644 --- a/src/wallet/wallet_args.h +++ b/src/wallet/wallet_args.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 35862bda1..6ebaaa395 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -699,26 +699,43 @@ namespace tools explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_weight_limit) : transfer_error(std::move(loc), "transaction is too big") , m_tx(tx) + , m_tx_valid(true) + , m_tx_weight(cryptonote::get_transaction_weight(tx)) , m_tx_weight_limit(tx_weight_limit) { } + explicit tx_too_big(std::string&& loc, uint64_t tx_weight, uint64_t tx_weight_limit) + : transfer_error(std::move(loc), "transaction would be too big") + , m_tx_valid(false) + , m_tx_weight(tx_weight) + , m_tx_weight_limit(tx_weight_limit) + { + } + + bool tx_valid() const { return m_tx_valid; } const cryptonote::transaction& tx() const { return m_tx; } + uint64_t tx_weight() const { return m_tx_weight; } uint64_t tx_weight_limit() const { return m_tx_weight_limit; } std::string to_string() const { std::ostringstream ss; - cryptonote::transaction tx = m_tx; ss << transfer_error::to_string() << ", tx_weight_limit = " << m_tx_weight_limit << - ", tx weight = " << get_transaction_weight(m_tx) << - ", tx:\n" << cryptonote::obj_to_json_str(tx); + ", tx weight = " << m_tx_weight; + if (m_tx_valid) + { + cryptonote::transaction tx = m_tx; + ss << ", tx:\n" << cryptonote::obj_to_json_str(tx); + } return ss.str(); } private: cryptonote::transaction m_tx; + bool m_tx_valid; + uint64_t m_tx_weight; uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index d7dc2914e..7040597df 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -63,6 +63,11 @@ namespace const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false}; const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"}; const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false}; + const command_line::arg_descriptor<std::string> arg_rpc_ssl = {"rpc-ssl", tools::wallet2::tr("Enable SSL on wallet RPC connections: enabled|disabled|autodetect"), "autodetect"}; + const command_line::arg_descriptor<std::string> arg_rpc_ssl_private_key = {"rpc-ssl-private-key", tools::wallet2::tr("Path to a PEM format private key"), ""}; + const command_line::arg_descriptor<std::string> arg_rpc_ssl_certificate = {"rpc-ssl-certificate", tools::wallet2::tr("Path to a PEM format certificate"), ""}; + const command_line::arg_descriptor<std::vector<std::string>> arg_rpc_ssl_allowed_certificates = {"rpc-ssl-allowed-certificates", tools::wallet2::tr("List of paths to PEM format certificates of allowed RPC servers (all allowed if empty)")}; + const command_line::arg_descriptor<std::vector<std::string>> arg_rpc_ssl_allowed_fingerprints = {"rpc-ssl-allowed-fingerprints", tools::wallet2::tr("List of certificate fingerprints to allow")}; constexpr const char default_rpc_username[] = "monero"; @@ -233,10 +238,36 @@ namespace tools assert(bool(http_login)); } // end auth enabled + auto rpc_ssl_private_key = command_line::get_arg(vm, arg_rpc_ssl_private_key); + auto rpc_ssl_certificate = command_line::get_arg(vm, arg_rpc_ssl_certificate); + auto rpc_ssl_allowed_certificates = command_line::get_arg(vm, arg_rpc_ssl_allowed_certificates); + auto rpc_ssl_allowed_fingerprints = command_line::get_arg(vm, arg_rpc_ssl_allowed_fingerprints); + auto rpc_ssl = command_line::get_arg(vm, arg_rpc_ssl); + epee::net_utils::ssl_support_t rpc_ssl_support; + if (!epee::net_utils::ssl_support_from_string(rpc_ssl_support, rpc_ssl)) + { + MERROR("Invalid argument for " << std::string(arg_rpc_ssl.name)); + return false; + } + std::list<std::string> allowed_certificates; + for (const std::string &path: rpc_ssl_allowed_certificates) + { + allowed_certificates.push_back({}); + if (!epee::file_io_utils::load_file_to_string(path, allowed_certificates.back())) + { + MERROR("Failed to load certificate: " << path); + allowed_certificates.back() = std::string(); + } + } + + std::vector<std::vector<uint8_t>> allowed_fingerprints{ rpc_ssl_allowed_fingerprints.size() }; + std::transform(rpc_ssl_allowed_fingerprints.begin(), rpc_ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); + m_net_server.set_threads_prefix("RPC"); auto rng = [](size_t len, uint8_t *ptr) { return crypto::rand(len, ptr); }; return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init( - rng, std::move(bind_port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login) + rng, std::move(bind_port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), + rpc_ssl_support, std::make_pair(rpc_ssl_private_key, rpc_ssl_certificate), std::move(allowed_certificates), std::move(allowed_fingerprints) ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -332,35 +363,59 @@ namespace tools set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try { - res.balance = m_wallet->balance(req.account_index); - res.unlocked_balance = m_wallet->unlocked_balance(req.account_index); + res.balance = req.all_accounts ? m_wallet->balance_all() : m_wallet->balance(req.account_index); + res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all() : m_wallet->unlocked_balance(req.account_index); res.multisig_import_needed = m_wallet->multisig() && m_wallet->has_multisig_partial_key_images(); - std::map<uint32_t, uint64_t> balance_per_subaddress = m_wallet->balance_per_subaddress(req.account_index); - std::map<uint32_t, uint64_t> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(req.account_index); - std::vector<tools::wallet2::transfer_details> transfers; - m_wallet->get_transfers(transfers); - std::set<uint32_t> address_indices = req.address_indices; - if (address_indices.empty()) + std::map<uint32_t, std::map<uint32_t, uint64_t>> balance_per_subaddress_per_account; + std::map<uint32_t, std::map<uint32_t, uint64_t>> unlocked_balance_per_subaddress_per_account; + if (req.all_accounts) { - for (const auto& i : balance_per_subaddress) - address_indices.insert(i.first); + for (uint32_t account_index = 0; account_index < m_wallet->get_num_subaddress_accounts(); ++account_index) + { + balance_per_subaddress_per_account[account_index] = m_wallet->balance_per_subaddress(account_index); + unlocked_balance_per_subaddress_per_account[account_index] = m_wallet->unlocked_balance_per_subaddress(account_index); + } } - for (uint32_t i : address_indices) + else { - wallet_rpc::COMMAND_RPC_GET_BALANCE::per_subaddress_info info; - info.address_index = i; - cryptonote::subaddress_index index = {req.account_index, info.address_index}; - info.address = m_wallet->get_subaddress_as_str(index); - info.balance = balance_per_subaddress[i]; - info.unlocked_balance = unlocked_balance_per_subaddress[i]; - info.label = m_wallet->get_subaddress_label(index); - info.num_unspent_outputs = std::count_if(transfers.begin(), transfers.end(), [&](const tools::wallet2::transfer_details& td) { return !td.m_spent && td.m_subaddr_index == index; }); - res.per_subaddress.push_back(info); + balance_per_subaddress_per_account[req.account_index] = m_wallet->balance_per_subaddress(req.account_index); + unlocked_balance_per_subaddress_per_account[req.account_index] = m_wallet->unlocked_balance_per_subaddress(req.account_index); + } + std::vector<tools::wallet2::transfer_details> transfers; + m_wallet->get_transfers(transfers); + for (const auto& p : balance_per_subaddress_per_account) + { + uint32_t account_index = p.first; + std::map<uint32_t, uint64_t> balance_per_subaddress = p.second; + std::map<uint32_t, uint64_t> unlocked_balance_per_subaddress = unlocked_balance_per_subaddress_per_account[account_index]; + std::set<uint32_t> address_indices; + if (!req.all_accounts && !req.address_indices.empty()) + { + address_indices = req.address_indices; + } + else + { + for (const auto& i : balance_per_subaddress) + address_indices.insert(i.first); + } + for (uint32_t i : address_indices) + { + wallet_rpc::COMMAND_RPC_GET_BALANCE::per_subaddress_info info; + info.account_index = account_index; + info.address_index = i; + cryptonote::subaddress_index index = {info.account_index, info.address_index}; + info.address = m_wallet->get_subaddress_as_str(index); + info.balance = balance_per_subaddress[i]; + info.unlocked_balance = unlocked_balance_per_subaddress[i]; + info.label = m_wallet->get_subaddress_label(index); + info.num_unspent_outputs = std::count_if(transfers.begin(), transfers.end(), [&](const tools::wallet2::transfer_details& td) { return !td.m_spent && td.m_subaddr_index == index; }); + res.per_subaddress.emplace_back(std::move(info)); + } } } catch (const std::exception& e) @@ -371,7 +426,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -411,7 +466,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); cryptonote::address_parse_info info; @@ -432,7 +487,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -449,7 +504,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -464,7 +519,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_accounts(const wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_accounts(const wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -503,7 +558,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_create_account(const wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_create_account(const wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -520,7 +575,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_label_account(const wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_label_account(const wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -535,7 +590,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); const std::pair<std::map<std::string, std::string>, std::vector<std::string>> account_tags = m_wallet->get_account_tags(); @@ -554,7 +609,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -569,7 +624,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -584,7 +639,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -599,7 +654,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getheight(const wallet_rpc::COMMAND_RPC_GET_HEIGHT::request& req, wallet_rpc::COMMAND_RPC_GET_HEIGHT::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_getheight(const wallet_rpc::COMMAND_RPC_GET_HEIGHT::request& req, wallet_rpc::COMMAND_RPC_GET_HEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -644,9 +699,11 @@ namespace tools return false; } + de.original = it->address; de.addr = info.address; de.is_subaddress = info.is_subaddress; de.amount = it->amount; + de.is_integrated = info.has_payment_id; dsts.push_back(de); if (info.has_payment_id) @@ -689,13 +746,9 @@ namespace tools if (wallet2::parse_long_payment_id(payment_id_str, long_payment_id)) { cryptonote::set_payment_id_to_tx_extra_nonce(extra_nonce, long_payment_id); } - /* or short payment ID */ - else if (wallet2::parse_short_payment_id(payment_id_str, short_payment_id)) { - cryptonote::set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, short_payment_id); - } else { er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment id has invalid format: \"" + payment_id_str + "\", expected 16 or 64 character string"; + er.message = "Payment id has invalid format: \"" + payment_id_str + "\", expected 64 character string"; return false; } @@ -811,7 +864,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx) { std::vector<cryptonote::tx_destination_entry> dsts; @@ -872,7 +925,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { std::vector<cryptonote::tx_destination_entry> dsts; @@ -919,7 +972,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1000,7 +1053,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1021,29 +1074,59 @@ namespace tools er.message = "command not supported by watch-only wallet"; return false; } - - tools::wallet2::unsigned_tx_set exported_txs; - try + if(req.unsigned_txset.empty() && req.multisig_txset.empty()) { - cryptonote::blobdata blob; - if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) - { - er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; - er.message = "Failed to parse hex."; - return false; + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "no txset provided"; + return false; + } + + std::vector <wallet2::tx_construction_data> tx_constructions; + if (!req.unsigned_txset.empty()) { + try { + tools::wallet2::unsigned_tx_set exported_txs; + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + if (!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "cannot load unsigned_txset"; + return false; + } + tx_constructions = exported_txs.txes; } - if(!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) - { + catch (const std::exception &e) { er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; - er.message = "cannot load unsigned_txset"; + er.message = "failed to parse unsigned transfers: " + std::string(e.what()); + return false; + } + } else if (!req.multisig_txset.empty()) { + try { + tools::wallet2::multisig_tx_set exported_txs; + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.multisig_txset, blob)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + if (!m_wallet->parse_multisig_tx_from_str(blob, exported_txs)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_MULTISIG_TX_DATA; + er.message = "cannot load multisig_txset"; + return false; + } + + for (size_t n = 0; n < exported_txs.m_ptx.size(); ++n) { + tx_constructions.push_back(exported_txs.m_ptx[n].construction_data); + } + } + catch (const std::exception &e) { + er.code = WALLET_RPC_ERROR_CODE_BAD_MULTISIG_TX_DATA; + er.message = "failed to parse multisig transfers: " + std::string(e.what()); return false; } - } - catch (const std::exception &e) - { - er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; - er.message = "failed to parse unsigned transfers: " + std::string(e.what()); - return false; } std::vector<tools::wallet2::pending_tx> ptx; @@ -1052,9 +1135,9 @@ namespace tools // gather info to ask the user std::unordered_map<cryptonote::account_public_address, std::pair<std::string, uint64_t>> dests; int first_known_non_zero_change_index = -1; - for (size_t n = 0; n < exported_txs.txes.size(); ++n) + for (size_t n = 0; n < tx_constructions.size(); ++n) { - const tools::wallet2::tx_construction_data &cd = exported_txs.txes[n]; + const tools::wallet2::tx_construction_data &cd = tx_constructions[n]; res.desc.push_back({0, 0, std::numeric_limits<uint32_t>::max(), 0, {}, "", 0, "", 0, 0, ""}); wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::transfer_description &desc = res.desc.back(); @@ -1118,7 +1201,7 @@ namespace tools { if (first_known_non_zero_change_index == -1) first_known_non_zero_change_index = n; - const tools::wallet2::tx_construction_data &cdn = exported_txs.txes[first_known_non_zero_change_index]; + const tools::wallet2::tx_construction_data &cdn = tx_constructions[first_known_non_zero_change_index]; if (memcmp(&cd.change_dts.addr, &cdn.change_dts.addr, sizeof(cd.change_dts.addr))) { er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; @@ -1146,7 +1229,7 @@ namespace tools if (desc.change_amount > 0) { - const tools::wallet2::tx_construction_data &cd0 = exported_txs.txes[0]; + const tools::wallet2::tx_construction_data &cd0 = tx_constructions[0]; desc.change_address = get_account_address_as_str(m_wallet->nettype(), cd0.subaddr_account > 0, cd0.change_dts.addr); } @@ -1165,7 +1248,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1225,7 +1308,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1250,7 +1333,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er, const connection_context *ctx) { std::vector<cryptonote::tx_destination_entry> dsts; std::vector<uint8_t> extra; @@ -1305,7 +1388,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_sweep_single(const wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::request& req, wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_sweep_single(const wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::request& req, wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::response& res, epee::json_rpc::error& er, const connection_context *ctx) { std::vector<cryptonote::tx_destination_entry> dsts; std::vector<uint8_t> extra; @@ -1394,7 +1477,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_relay_tx(const wallet_rpc::COMMAND_RPC_RELAY_TX::request& req, wallet_rpc::COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_relay_tx(const wallet_rpc::COMMAND_RPC_RELAY_TX::request& req, wallet_rpc::COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -1436,7 +1519,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_make_integrated_address(const wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_make_integrated_address(const wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -1500,7 +1583,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_split_integrated_address(const wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_split_integrated_address(const wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try @@ -1531,7 +1614,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1553,7 +1636,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); crypto::hash payment_id; @@ -1602,7 +1685,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { res.payments.clear(); if (!m_wallet) return not_open(er); @@ -1636,23 +1719,14 @@ namespace tools cryptonote::blobdata payment_id_blob; // TODO - should the whole thing fail because of one bad id? - - if(!epee::string_tools::parse_hexstr_to_binbuff(payment_id_str, payment_id_blob)) - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment ID has invalid format: " + payment_id_str; - return false; - } - - if(sizeof(payment_id) == payment_id_blob.size()) + bool r; + if (payment_id_str.size() == 2 * sizeof(payment_id)) { - payment_id = *reinterpret_cast<const crypto::hash*>(payment_id_blob.data()); + r = epee::string_tools::hex_to_pod(payment_id_str, payment_id); } - else if(sizeof(payment_id8) == payment_id_blob.size()) + else if (payment_id_str.size() == 2 * sizeof(payment_id8)) { - payment_id8 = *reinterpret_cast<const crypto::hash8*>(payment_id_blob.data()); - memcpy(payment_id.data, payment_id8.data, 8); - memset(payment_id.data + 8, 0, 24); + r = epee::string_tools::hex_to_pod(payment_id_str, payment_id8); } else { @@ -1661,6 +1735,13 @@ namespace tools return false; } + if(!r) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment ID has invalid format: " + payment_id_str; + return false; + } + std::list<wallet2::payment_details> payment_list; m_wallet->get_payments(payment_id, payment_list, req.min_block_height); @@ -1681,7 +1762,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if(req.transfer_type.compare("all") != 0 && req.transfer_type.compare("available") != 0 && req.transfer_type.compare("unavailable") != 0) @@ -1733,7 +1814,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1804,7 +1885,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1826,7 +1907,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1840,7 +1921,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1875,7 +1956,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1898,7 +1979,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1941,7 +2022,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er, const connection_context *ctx) { res.notes.clear(); if (!m_wallet) return not_open(er); @@ -1970,7 +2051,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_set_attribute(const wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_set_attribute(const wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1985,7 +2066,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_attribute(const wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_attribute(const wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -1998,7 +2079,7 @@ namespace tools res.value = m_wallet->get_attribute(req.key); return true; } - bool wallet_rpc_server::on_get_tx_key(const wallet_rpc::COMMAND_RPC_GET_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_GET_TX_KEY::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_tx_key(const wallet_rpc::COMMAND_RPC_GET_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_GET_TX_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2027,7 +2108,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_check_tx_key(const wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_check_tx_key(const wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2089,7 +2170,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_tx_proof(const wallet_rpc::COMMAND_RPC_GET_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_TX_PROOF::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_tx_proof(const wallet_rpc::COMMAND_RPC_GET_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_TX_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2122,7 +2203,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_check_tx_proof(const wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_check_tx_proof(const wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2144,9 +2225,6 @@ namespace tools try { - uint64_t received; - bool in_pool; - uint64_t confirmations; res.good = m_wallet->check_tx_proof(txid, info.address, info.is_subaddress, req.message, req.signature, res.received, res.in_pool, res.confirmations); } catch (const std::exception &e) @@ -2158,7 +2236,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_spend_proof(const wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_spend_proof(const wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2183,7 +2261,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_check_spend_proof(const wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_check_spend_proof(const wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2208,7 +2286,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_reserve_proof(const wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_reserve_proof(const wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2237,7 +2315,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_check_reserve_proof(const wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_check_reserve_proof(const wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2268,7 +2346,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2285,10 +2363,18 @@ namespace tools max_height = req.max_height <= max_height ? req.max_height : max_height; } + boost::optional<uint32_t> account_index = req.account_index; + std::set<uint32_t> subaddr_indices = req.subaddr_indices; + if (req.all_accounts) + { + account_index = boost::none; + subaddr_indices.clear(); + } + if (req.in) { std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; - m_wallet->get_payments(payments, min_height, max_height, req.account_index, req.subaddr_indices); + m_wallet->get_payments(payments, min_height, max_height, account_index, subaddr_indices); for (std::list<std::pair<crypto::hash, tools::wallet2::payment_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { res.in.push_back(wallet_rpc::transfer_entry()); fill_transfer_entry(res.in.back(), i->second.m_tx_hash, i->first, i->second); @@ -2298,7 +2384,7 @@ namespace tools if (req.out) { std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>> payments; - m_wallet->get_payments_out(payments, min_height, max_height, req.account_index, req.subaddr_indices); + m_wallet->get_payments_out(payments, min_height, max_height, account_index, subaddr_indices); for (std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { res.out.push_back(wallet_rpc::transfer_entry()); fill_transfer_entry(res.out.back(), i->first, i->second); @@ -2307,7 +2393,7 @@ namespace tools if (req.pending || req.failed) { std::list<std::pair<crypto::hash, tools::wallet2::unconfirmed_transfer_details>> upayments; - m_wallet->get_unconfirmed_payments_out(upayments, req.account_index, req.subaddr_indices); + m_wallet->get_unconfirmed_payments_out(upayments, account_index, subaddr_indices); for (std::list<std::pair<crypto::hash, tools::wallet2::unconfirmed_transfer_details>>::const_iterator i = upayments.begin(); i != upayments.end(); ++i) { const tools::wallet2::unconfirmed_transfer_details &pd = i->second; bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; @@ -2324,7 +2410,7 @@ namespace tools m_wallet->update_pool_state(); std::list<std::pair<crypto::hash, tools::wallet2::pool_payment_details>> payments; - m_wallet->get_unconfirmed_payments(payments, req.account_index, req.subaddr_indices); + m_wallet->get_unconfirmed_payments(payments, account_index, subaddr_indices); for (std::list<std::pair<crypto::hash, tools::wallet2::pool_payment_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { res.pool.push_back(wallet_rpc::transfer_entry()); fill_transfer_entry(res.pool.back(), i->first, i->second); @@ -2334,7 +2420,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2424,7 +2510,7 @@ namespace tools return false; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2453,7 +2539,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2490,12 +2576,12 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); try { - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); res.offset = ski.first; res.signed_key_images.resize(ski.second.size()); for (size_t n = 0; n < ski.second.size(); ++n) @@ -2514,7 +2600,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2535,23 +2621,19 @@ namespace tools ski.resize(req.signed_key_images.size()); for (size_t n = 0; n < ski.size(); ++n) { - cryptonote::blobdata bd; - - if(!epee::string_tools::parse_hexstr_to_binbuff(req.signed_key_images[n].key_image, bd) || bd.size() != sizeof(crypto::key_image)) + if (!epee::string_tools::hex_to_pod(req.signed_key_images[n].key_image, ski[n].first)) { er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY_IMAGE; er.message = "failed to parse key image"; return false; } - ski[n].first = *reinterpret_cast<const crypto::key_image*>(bd.data()); - if(!epee::string_tools::parse_hexstr_to_binbuff(req.signed_key_images[n].signature, bd) || bd.size() != sizeof(crypto::signature)) + if (!epee::string_tools::hex_to_pod(req.signed_key_images[n].signature, ski[n].second)) { er.code = WALLET_RPC_ERROR_CODE_WRONG_SIGNATURE; er.message = "failed to parse signature"; return false; } - ski[n].second = *reinterpret_cast<const crypto::signature*>(bd.data()); } uint64_t spent = 0, unspent = 0; uint64_t height = m_wallet->import_key_images(ski, req.offset, spent, unspent); @@ -2569,7 +2651,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); std::string error; @@ -2585,7 +2667,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_parse_uri(const wallet_rpc::COMMAND_RPC_PARSE_URI::request& req, wallet_rpc::COMMAND_RPC_PARSE_URI::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_parse_uri(const wallet_rpc::COMMAND_RPC_PARSE_URI::request& req, wallet_rpc::COMMAND_RPC_PARSE_URI::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); std::string error; @@ -2598,7 +2680,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); const auto ab = m_wallet->get_address_book(); @@ -2625,7 +2707,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2700,7 +2782,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2726,7 +2808,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2748,7 +2830,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -2770,7 +2852,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (!m_wallet->is_trusted_daemon()) @@ -2805,7 +2887,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); cryptonote::COMMAND_RPC_STOP_MINING::request daemon_req; @@ -2820,13 +2902,13 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx) { crypto::ElectrumWords::get_language_list(res.languages); return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (m_wallet_dir.empty()) { @@ -2855,8 +2937,6 @@ namespace tools std::vector<std::string> languages; crypto::ElectrumWords::get_language_list(languages); std::vector<std::string>::iterator it; - std::string wallet_file; - char *ptr; it = std::find(languages.begin(), languages.end(), req.language); if (it == languages.end()) @@ -2926,7 +3006,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (m_wallet_dir.empty()) { @@ -2996,7 +3076,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -3014,7 +3094,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3127,7 +3207,175 @@ namespace tools } } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request &req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response &res, epee::json_rpc::error &er) + bool wallet_rpc_server::on_generate_from_keys(const wallet_rpc::COMMAND_RPC_GENERATE_FROM_KEYS::request &req, wallet_rpc::COMMAND_RPC_GENERATE_FROM_KEYS::response &res, epee::json_rpc::error &er, const connection_context *ctx) + { + if (m_wallet_dir.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_NO_WALLET_DIR; + er.message = "No wallet dir configured"; + return false; + } + + // early check for mandatory fields + if (req.filename.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; + return false; + } + if (req.viewkey.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'viewkey' is mandatory. Please provide a view key you want to restore from."; + return false; + } + if (req.address.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'address' is mandatory. Please provide a public address."; + return false; + } + + namespace po = boost::program_options; + po::variables_map vm2; + const char *ptr = strchr(req.filename.c_str(), '/'); + #ifdef _WIN32 + if (!ptr) + ptr = strchr(req.filename.c_str(), '\\'); + if (!ptr) + ptr = strchr(req.filename.c_str(), ':'); + #endif + if (ptr) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Invalid filename"; + return false; + } + std::string wallet_file = m_wallet_dir + "/" + req.filename; + // check if wallet file already exists + if (!wallet_file.empty()) + { + try + { + boost::system::error_code ignored_ec; + THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(wallet_file, ignored_ec), error::file_exists, wallet_file); + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet already exists."; + return false; + } + } + + { + po::options_description desc("dummy"); + const command_line::arg_descriptor<std::string, true> arg_password = {"password", "password"}; + const char *argv[4]; + int argc = 3; + argv[0] = "wallet-rpc"; + argv[1] = "--password"; + argv[2] = req.password.c_str(); + argv[3] = NULL; + vm2 = *m_vm; + command_line::add_arg(desc, arg_password); + po::store(po::parse_command_line(argc, argv, desc), vm2); + } + + auto rc = tools::wallet2::make_new(vm2, true, nullptr); + std::unique_ptr<wallet2> wal; + wal = std::move(rc.first); + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to create wallet"; + return false; + } + + cryptonote::address_parse_info info; + if(!get_account_address_from_str(info, wal->nettype(), req.address)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to parse public address"; + return false; + } + + epee::wipeable_string password = rc.second.password(); + epee::wipeable_string viewkey_string = req.viewkey; + crypto::secret_key viewkey; + if (!viewkey_string.hex_to_pod(unwrap(unwrap(viewkey)))) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to parse view key secret key"; + return false; + } + + try + { + if (!req.spendkey.empty()) + { + epee::wipeable_string spendkey_string = req.spendkey; + crypto::secret_key spendkey; + if (!spendkey_string.hex_to_pod(unwrap(unwrap(spendkey)))) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to parse spend key secret key"; + return false; + } + wal->generate(wallet_file, std::move(rc.second).password(), info.address, spendkey, viewkey, false); + res.info = "Wallet has been generated successfully."; + } + else + { + wal->generate(wallet_file, std::move(rc.second).password(), info.address, viewkey, false); + res.info = "Watch-only wallet has been generated successfully."; + } + MINFO("Wallet has been generated.\n"); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to generate wallet"; + return false; + } + + // set blockheight if given + try + { + wal->set_refresh_from_block_height(req.restore_height); + wal->rewrite(wallet_file, password); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + } + m_wallet = wal.release(); + res.address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request &req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response &res, epee::json_rpc::error &er, const connection_context *ctx) { if (m_wallet_dir.empty()) { @@ -3321,14 +3569,14 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); res.multisig = m_wallet->multisig(&res.ready, &res.threshold, &res.total); return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3354,7 +3602,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3391,7 +3639,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3431,7 +3679,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3504,7 +3752,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3555,7 +3803,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3604,7 +3852,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3673,7 +3921,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); if (m_restricted) @@ -3739,7 +3987,58 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er) + bool wallet_rpc_server::on_validate_address(const wallet_rpc::COMMAND_RPC_VALIDATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_VALIDATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + cryptonote::address_parse_info info; + static const struct { cryptonote::network_type type; const char *stype; } net_types[] = { + { cryptonote::MAINNET, "mainnet" }, + { cryptonote::TESTNET, "testnet" }, + { cryptonote::STAGENET, "stagenet" }, + }; + for (const auto &net_type: net_types) + { + if (!req.any_net_type && net_type.type != m_wallet->nettype()) + continue; + if (req.allow_openalias) + { + std::string address; + res.valid = get_account_address_from_str_or_url(info, net_type.type, req.address, + [&er, &address](const std::string &url, const std::vector<std::string> &addresses, bool dnssec_valid)->std::string { + if (!dnssec_valid) + { + er.message = std::string("Invalid DNSSEC for ") + url; + return {}; + } + if (addresses.empty()) + { + er.message = std::string("No Monero address found at ") + url; + return {}; + } + address = addresses[0]; + return address; + }); + if (res.valid) + res.openalias_address = address; + } + else + { + res.valid = cryptonote::get_account_address_from_str(info, net_type.type, req.address); + } + if (res.valid) + { + res.integrated = info.has_payment_id; + res.subaddress = info.is_subaddress; + res.nettype = net_type.stype; + return true; + } + } + + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = std::string("Invalid address"); + return false; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er, const connection_context *ctx) { res.version = WALLET_RPC_VERSION; return true; @@ -3935,6 +4234,11 @@ int main(int argc, char** argv) { command_line::add_arg(desc_params, arg_from_json); command_line::add_arg(desc_params, arg_wallet_dir); command_line::add_arg(desc_params, arg_prompt_for_password); + command_line::add_arg(desc_params, arg_rpc_ssl); + command_line::add_arg(desc_params, arg_rpc_ssl_private_key); + command_line::add_arg(desc_params, arg_rpc_ssl_certificate); + command_line::add_arg(desc_params, arg_rpc_ssl_allowed_certificates); + command_line::add_arg(desc_params, arg_rpc_ssl_allowed_fingerprints); daemonizer::init_options(hidden_options, desc_params); desc_params.add(hidden_options); diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index abbbe82c5..affaf10f7 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -137,6 +137,7 @@ namespace tools MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) + MAP_JON_RPC_WE("generate_from_keys", on_generate_from_keys, wallet_rpc::COMMAND_RPC_GENERATE_FROM_KEYS) MAP_JON_RPC_WE("restore_deterministic_wallet", on_restore_deterministic_wallet, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) @@ -147,90 +148,92 @@ namespace tools MAP_JON_RPC_WE("exchange_multisig_keys", on_exchange_multisig_keys, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS) MAP_JON_RPC_WE("sign_multisig", on_sign_multisig, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG) MAP_JON_RPC_WE("submit_multisig", on_submit_multisig, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG) + MAP_JON_RPC_WE("validate_address", on_validate_address, wallet_rpc::COMMAND_RPC_VALIDATE_ADDRESS) MAP_JON_RPC_WE("get_version", on_get_version, wallet_rpc::COMMAND_RPC_GET_VERSION) END_JSON_RPC_MAP() END_URI_MAP2() //json_rpc - bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er); - bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er); - bool on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er); - bool on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er); - bool on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er); - bool on_get_accounts(const wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::response& res, epee::json_rpc::error& er); - bool on_create_account(const wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::response& res, epee::json_rpc::error& er); - bool on_label_account(const wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::response& res, epee::json_rpc::error& er); - bool on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er); - bool on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er); - bool on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er); - bool on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er); - bool on_getheight(const wallet_rpc::COMMAND_RPC_GET_HEIGHT::request& req, wallet_rpc::COMMAND_RPC_GET_HEIGHT::response& res, epee::json_rpc::error& er); - bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); - bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er); - bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er); - bool on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er); - bool on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er); - bool on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er); - bool on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er); - bool on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er); - bool on_sweep_single(const wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::request& req, wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::response& res, epee::json_rpc::error& er); - bool on_relay_tx(const wallet_rpc::COMMAND_RPC_RELAY_TX::request& req, wallet_rpc::COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& er); - bool on_make_integrated_address(const wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er); - bool on_split_integrated_address(const wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er); - bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er); - bool on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er); - bool on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er); - bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er); - bool on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er); - bool on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er); - bool on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er); - bool on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er); - bool on_set_attribute(const wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::response& res, epee::json_rpc::error& er); - bool on_get_attribute(const wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::response& res, epee::json_rpc::error& er); - bool on_get_tx_key(const wallet_rpc::COMMAND_RPC_GET_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_GET_TX_KEY::response& res, epee::json_rpc::error& er); - bool on_check_tx_key(const wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::response& res, epee::json_rpc::error& er); - bool on_get_tx_proof(const wallet_rpc::COMMAND_RPC_GET_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_TX_PROOF::response& res, epee::json_rpc::error& er); - bool on_check_tx_proof(const wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::response& res, epee::json_rpc::error& er); - bool on_get_spend_proof(const wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::response& res, epee::json_rpc::error& er); - bool on_check_spend_proof(const wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::response& res, epee::json_rpc::error& er); - bool on_get_reserve_proof(const wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::response& res, epee::json_rpc::error& er); - bool on_check_reserve_proof(const wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::response& res, epee::json_rpc::error& er); - bool on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er); - bool on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er); - bool on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er); - bool on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er); - bool on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er); - bool on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er); - bool on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er); - bool on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er); - bool on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er); - bool on_parse_uri(const wallet_rpc::COMMAND_RPC_PARSE_URI::request& req, wallet_rpc::COMMAND_RPC_PARSE_URI::response& res, epee::json_rpc::error& er); - bool on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); - bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); - bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); - bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er); - bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er); - bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er); - bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er); - bool on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er); - bool on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er); - bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); - bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); - bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); - bool on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request& req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response& res, epee::json_rpc::error& er); - bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er); - bool on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er); - bool on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er); + bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_accounts(const wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_create_account(const wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_CREATE_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_label_account(const wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::request& req, wallet_rpc::COMMAND_RPC_LABEL_ACCOUNT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_getheight(const wallet_rpc::COMMAND_RPC_GET_HEIGHT::request& req, wallet_rpc::COMMAND_RPC_GET_HEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_sweep_single(const wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::request& req, wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_relay_tx(const wallet_rpc::COMMAND_RPC_RELAY_TX::request& req, wallet_rpc::COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_make_integrated_address(const wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_MAKE_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_split_integrated_address(const wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_tx_notes(const wallet_rpc::COMMAND_RPC_GET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_GET_TX_NOTES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_set_attribute(const wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_attribute(const wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_tx_key(const wallet_rpc::COMMAND_RPC_GET_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_GET_TX_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_check_tx_key(const wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_tx_proof(const wallet_rpc::COMMAND_RPC_GET_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_TX_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_check_tx_proof(const wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_TX_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_spend_proof(const wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_SPEND_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_check_spend_proof(const wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_SPEND_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_reserve_proof(const wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_GET_RESERVE_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_check_reserve_proof(const wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::request& req, wallet_rpc::COMMAND_RPC_CHECK_RESERVE_PROOF::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_parse_uri(const wallet_rpc::COMMAND_RPC_PARSE_URI::request& req, wallet_rpc::COMMAND_RPC_PARSE_URI::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_generate_from_keys(const wallet_rpc::COMMAND_RPC_GENERATE_FROM_KEYS::request& req, wallet_rpc::COMMAND_RPC_GENERATE_FROM_KEYS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request& req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_validate_address(const wallet_rpc::COMMAND_RPC_VALIDATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_VALIDATE_ADDRESS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); //json rpc v2 - bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er); + bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); // helpers void fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &txid, const crypto::hash &payment_id, const tools::wallet2::payment_details &pd); @@ -245,6 +248,8 @@ namespace tools bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er); + bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); + wallet2 *m_wallet; std::string m_wallet_dir; tools::private_file rpc_login_file; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index afb8c6e91..b0e8bed93 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 6 +#define WALLET_RPC_VERSION_MINOR 8 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -59,18 +59,22 @@ namespace wallet_rpc struct COMMAND_RPC_GET_BALANCE { - struct request + struct request_t { uint32_t account_index; std::set<uint32_t> address_indices; + bool all_accounts; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) KV_SERIALIZE(address_indices) + KV_SERIALIZE_OPT(all_accounts, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct per_subaddress_info { + uint32_t account_index; uint32_t address_index; std::string address; uint64_t balance; @@ -79,6 +83,7 @@ namespace wallet_rpc uint64_t num_unspent_outputs; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(account_index) KV_SERIALIZE(address_index) KV_SERIALIZE(address) KV_SERIALIZE(balance) @@ -88,7 +93,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { uint64_t balance; uint64_t unlocked_balance; @@ -102,11 +107,12 @@ namespace wallet_rpc KV_SERIALIZE(per_subaddress) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ADDRESS { - struct request + struct request_t { uint32_t account_index; std::vector<uint32_t> address_index; @@ -115,6 +121,7 @@ namespace wallet_rpc KV_SERIALIZE(address_index) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct address_info { @@ -131,7 +138,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::string address; // to remain compatible with older RPC format std::vector<address_info> addresses; @@ -141,30 +148,33 @@ namespace wallet_rpc KV_SERIALIZE(addresses) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ADDRESS_INDEX { - struct request + struct request_t { std::string address; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { cryptonote::subaddress_index index; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(index) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CREATE_ADDRESS { - struct request + struct request_t { uint32_t account_index; std::string label; @@ -174,8 +184,9 @@ namespace wallet_rpc KV_SERIALIZE(label) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string address; uint32_t address_index; @@ -185,11 +196,12 @@ namespace wallet_rpc KV_SERIALIZE(address_index) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_LABEL_ADDRESS { - struct request + struct request_t { cryptonote::subaddress_index index; std::string label; @@ -199,17 +211,19 @@ namespace wallet_rpc KV_SERIALIZE(label) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ACCOUNTS { - struct request + struct request_t { std::string tag; // all accounts if empty, otherwise those accounts with this tag @@ -217,6 +231,7 @@ namespace wallet_rpc KV_SERIALIZE(tag) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct subaddress_account_info { @@ -237,7 +252,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { uint64_t total_balance; uint64_t total_unlocked_balance; @@ -249,19 +264,21 @@ namespace wallet_rpc KV_SERIALIZE(subaddress_accounts) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CREATE_ACCOUNT { - struct request + struct request_t { std::string label; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(label) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint32_t account_index; std::string address; // the 0-th address for convenience @@ -270,11 +287,12 @@ namespace wallet_rpc KV_SERIALIZE(address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_LABEL_ACCOUNT { - struct request + struct request_t { uint32_t account_index; std::string label; @@ -284,21 +302,24 @@ namespace wallet_rpc KV_SERIALIZE(label) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ACCOUNT_TAGS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct account_tag_info { @@ -313,7 +334,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::vector<account_tag_info> account_tags; @@ -321,11 +342,12 @@ namespace wallet_rpc KV_SERIALIZE(account_tags) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_TAG_ACCOUNTS { - struct request + struct request_t { std::string tag; std::set<uint32_t> accounts; @@ -335,17 +357,19 @@ namespace wallet_rpc KV_SERIALIZE(accounts) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_UNTAG_ACCOUNTS { - struct request + struct request_t { std::set<uint32_t> accounts; @@ -353,17 +377,19 @@ namespace wallet_rpc KV_SERIALIZE(accounts) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION { - struct request + struct request_t { std::string tag; std::string description; @@ -373,29 +399,33 @@ namespace wallet_rpc KV_SERIALIZE(description) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_HEIGHT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t height; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct transfer_destination @@ -410,7 +440,7 @@ namespace wallet_rpc struct COMMAND_RPC_TRANSFER { - struct request + struct request_t { std::list<transfer_destination> destinations; uint32_t account_index; @@ -440,8 +470,9 @@ namespace wallet_rpc KV_SERIALIZE_OPT(get_tx_metadata, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string tx_hash; std::string tx_key; @@ -463,11 +494,12 @@ namespace wallet_rpc KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_TRANSFER_SPLIT { - struct request + struct request_t { std::list<transfer_destination> destinations; uint32_t account_index; @@ -497,6 +529,7 @@ namespace wallet_rpc KV_SERIALIZE_OPT(get_tx_metadata, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct key_list { @@ -507,7 +540,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::list<std::string> tx_hash_list; std::list<std::string> tx_key_list; @@ -529,6 +562,7 @@ namespace wallet_rpc KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_DESCRIBE_TRANSFER @@ -573,16 +607,19 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct request + struct request_t { std::string unsigned_txset; + std::string multisig_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(multisig_txset) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<transfer_description> desc; @@ -590,11 +627,12 @@ namespace wallet_rpc KV_SERIALIZE(desc) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SIGN_TRANSFER { - struct request + struct request_t { std::string unsigned_txset; bool export_raw; @@ -606,8 +644,9 @@ namespace wallet_rpc KV_SERIALIZE_OPT(get_tx_keys, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string signed_txset; std::list<std::string> tx_hash_list; @@ -621,11 +660,12 @@ namespace wallet_rpc KV_SERIALIZE(tx_key_list) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SUBMIT_TRANSFER { - struct request + struct request_t { std::string tx_data_hex; @@ -633,8 +673,9 @@ namespace wallet_rpc KV_SERIALIZE(tx_data_hex) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<std::string> tx_hash_list; @@ -642,11 +683,12 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash_list) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SWEEP_DUST { - struct request + struct request_t { bool get_tx_keys; bool do_not_relay; @@ -660,6 +702,7 @@ namespace wallet_rpc KV_SERIALIZE_OPT(get_tx_metadata, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct key_list { @@ -670,7 +713,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::list<std::string> tx_hash_list; std::list<std::string> tx_key_list; @@ -692,11 +735,12 @@ namespace wallet_rpc KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SWEEP_ALL { - struct request + struct request_t { std::string address; uint32_t account_index; @@ -730,6 +774,7 @@ namespace wallet_rpc KV_SERIALIZE_OPT(get_tx_metadata, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct key_list { @@ -740,7 +785,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::list<std::string> tx_hash_list; std::list<std::string> tx_key_list; @@ -762,11 +807,12 @@ namespace wallet_rpc KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SWEEP_SINGLE { - struct request + struct request_t { std::string address; uint32_t priority; @@ -796,8 +842,9 @@ namespace wallet_rpc KV_SERIALIZE_OPT(get_tx_metadata, false) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string tx_hash; std::string tx_key; @@ -819,11 +866,12 @@ namespace wallet_rpc KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_RELAY_TX { - struct request + struct request_t { std::string hex; @@ -831,8 +879,9 @@ namespace wallet_rpc KV_SERIALIZE(hex) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string tx_hash; @@ -840,21 +889,24 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_STORE { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct payment_details @@ -880,7 +932,7 @@ namespace wallet_rpc struct COMMAND_RPC_GET_PAYMENTS { - struct request + struct request_t { std::string payment_id; @@ -888,8 +940,9 @@ namespace wallet_rpc KV_SERIALIZE(payment_id) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<payment_details> payments; @@ -897,11 +950,12 @@ namespace wallet_rpc KV_SERIALIZE(payments) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_BULK_PAYMENTS { - struct request + struct request_t { std::vector<std::string> payment_ids; uint64_t min_block_height; @@ -911,8 +965,9 @@ namespace wallet_rpc KV_SERIALIZE(min_block_height) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<payment_details> payments; @@ -920,6 +975,7 @@ namespace wallet_rpc KV_SERIALIZE(payments) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct transfer_details @@ -943,7 +999,7 @@ namespace wallet_rpc struct COMMAND_RPC_INCOMING_TRANSFERS { - struct request + struct request_t { std::string transfer_type; uint32_t account_index; @@ -955,8 +1011,9 @@ namespace wallet_rpc KV_SERIALIZE(subaddr_indices) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<transfer_details> transfers; @@ -964,12 +1021,13 @@ namespace wallet_rpc KV_SERIALIZE(transfers) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; //JSON RPC V2 struct COMMAND_RPC_QUERY_KEY { - struct request + struct request_t { std::string key_type; @@ -977,8 +1035,9 @@ namespace wallet_rpc KV_SERIALIZE(key_type) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string key; @@ -986,11 +1045,12 @@ namespace wallet_rpc KV_SERIALIZE(key) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_MAKE_INTEGRATED_ADDRESS { - struct request + struct request_t { std::string standard_address; std::string payment_id; @@ -1000,8 +1060,9 @@ namespace wallet_rpc KV_SERIALIZE(payment_id) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string integrated_address; std::string payment_id; @@ -1011,11 +1072,12 @@ namespace wallet_rpc KV_SERIALIZE(payment_id) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SPLIT_INTEGRATED_ADDRESS { - struct request + struct request_t { std::string integrated_address; @@ -1023,8 +1085,9 @@ namespace wallet_rpc KV_SERIALIZE(integrated_address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string standard_address; std::string payment_id; @@ -1036,26 +1099,29 @@ namespace wallet_rpc KV_SERIALIZE(is_subaddress) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_STOP_WALLET { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_RESCAN_BLOCKCHAIN { - struct request + struct request_t { bool hard; @@ -1063,17 +1129,19 @@ namespace wallet_rpc KV_SERIALIZE_OPT(hard, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SET_TX_NOTES { - struct request + struct request_t { std::list<std::string> txids; std::list<std::string> notes; @@ -1083,17 +1151,19 @@ namespace wallet_rpc KV_SERIALIZE(notes) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_TX_NOTES { - struct request + struct request_t { std::list<std::string> txids; @@ -1101,8 +1171,9 @@ namespace wallet_rpc KV_SERIALIZE(txids) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<std::string> notes; @@ -1110,11 +1181,12 @@ namespace wallet_rpc KV_SERIALIZE(notes) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SET_ATTRIBUTE { - struct request + struct request_t { std::string key; std::string value; @@ -1124,17 +1196,19 @@ namespace wallet_rpc KV_SERIALIZE(value) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ATTRIBUTE { - struct request + struct request_t { std::string key; @@ -1143,8 +1217,9 @@ namespace wallet_rpc KV_SERIALIZE(key) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string value; @@ -1152,11 +1227,12 @@ namespace wallet_rpc KV_SERIALIZE(value) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_TX_KEY { - struct request + struct request_t { std::string txid; @@ -1164,8 +1240,9 @@ namespace wallet_rpc KV_SERIALIZE(txid) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string tx_key; @@ -1173,11 +1250,12 @@ namespace wallet_rpc KV_SERIALIZE(tx_key) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CHECK_TX_KEY { - struct request + struct request_t { std::string txid; std::string tx_key; @@ -1189,8 +1267,9 @@ namespace wallet_rpc KV_SERIALIZE(address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t received; bool in_pool; @@ -1202,11 +1281,12 @@ namespace wallet_rpc KV_SERIALIZE(confirmations) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_TX_PROOF { - struct request + struct request_t { std::string txid; std::string address; @@ -1218,8 +1298,9 @@ namespace wallet_rpc KV_SERIALIZE(message) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string signature; @@ -1227,11 +1308,12 @@ namespace wallet_rpc KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CHECK_TX_PROOF { - struct request + struct request_t { std::string txid; std::string address; @@ -1245,8 +1327,9 @@ namespace wallet_rpc KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { bool good; uint64_t received; @@ -1260,6 +1343,7 @@ namespace wallet_rpc KV_SERIALIZE(confirmations) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct transfer_entry @@ -1301,7 +1385,7 @@ namespace wallet_rpc struct COMMAND_RPC_GET_SPEND_PROOF { - struct request + struct request_t { std::string txid; std::string message; @@ -1311,8 +1395,9 @@ namespace wallet_rpc KV_SERIALIZE(message) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string signature; @@ -1320,11 +1405,12 @@ namespace wallet_rpc KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CHECK_SPEND_PROOF { - struct request + struct request_t { std::string txid; std::string message; @@ -1336,8 +1422,9 @@ namespace wallet_rpc KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { bool good; @@ -1345,11 +1432,12 @@ namespace wallet_rpc KV_SERIALIZE(good) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_RESERVE_PROOF { - struct request + struct request_t { bool all; uint32_t account_index; // ignored when `all` is true @@ -1363,8 +1451,9 @@ namespace wallet_rpc KV_SERIALIZE(message) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string signature; @@ -1372,11 +1461,12 @@ namespace wallet_rpc KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CHECK_RESERVE_PROOF { - struct request + struct request_t { std::string address; std::string message; @@ -1388,8 +1478,9 @@ namespace wallet_rpc KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { bool good; uint64_t total; @@ -1401,11 +1492,12 @@ namespace wallet_rpc KV_SERIALIZE(spent) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_TRANSFERS { - struct request + struct request_t { bool in; bool out; @@ -1418,6 +1510,7 @@ namespace wallet_rpc uint64_t max_height; uint32_t account_index; std::set<uint32_t> subaddr_indices; + bool all_accounts; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(in); @@ -1430,10 +1523,12 @@ namespace wallet_rpc KV_SERIALIZE_OPT(max_height, (uint64_t)CRYPTONOTE_MAX_BLOCK_NUMBER); KV_SERIALIZE(account_index); KV_SERIALIZE(subaddr_indices); + KV_SERIALIZE_OPT(all_accounts, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<transfer_entry> in; std::list<transfer_entry> out; @@ -1449,11 +1544,12 @@ namespace wallet_rpc KV_SERIALIZE(pool); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_TRANSFER_BY_TXID { - struct request + struct request_t { std::string txid; uint32_t account_index; @@ -1463,8 +1559,9 @@ namespace wallet_rpc KV_SERIALIZE_OPT(account_index, (uint32_t)0) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { transfer_entry transfer; std::list<transfer_entry> transfers; @@ -1474,11 +1571,12 @@ namespace wallet_rpc KV_SERIALIZE(transfers); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SIGN { - struct request + struct request_t { std::string data; @@ -1486,8 +1584,9 @@ namespace wallet_rpc KV_SERIALIZE(data); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string signature; @@ -1495,11 +1594,12 @@ namespace wallet_rpc KV_SERIALIZE(signature); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_VERIFY { - struct request + struct request_t { std::string data; std::string address; @@ -1511,8 +1611,9 @@ namespace wallet_rpc KV_SERIALIZE(signature); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { bool good; @@ -1520,17 +1621,19 @@ namespace wallet_rpc KV_SERIALIZE(good); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_EXPORT_OUTPUTS { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string outputs_data_hex; @@ -1538,11 +1641,12 @@ namespace wallet_rpc KV_SERIALIZE(outputs_data_hex); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_IMPORT_OUTPUTS { - struct request + struct request_t { std::string outputs_data_hex; @@ -1550,8 +1654,9 @@ namespace wallet_rpc KV_SERIALIZE(outputs_data_hex); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t num_imported; @@ -1559,15 +1664,20 @@ namespace wallet_rpc KV_SERIALIZE(num_imported); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_EXPORT_KEY_IMAGES { - struct request + struct request_t { + bool all; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(all, false); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct signed_key_image { @@ -1580,7 +1690,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { uint32_t offset; std::vector<signed_key_image> signed_key_images; @@ -1590,6 +1700,7 @@ namespace wallet_rpc KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_IMPORT_KEY_IMAGES @@ -1605,7 +1716,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct request + struct request_t { uint32_t offset; std::vector<signed_key_image> signed_key_images; @@ -1615,8 +1726,9 @@ namespace wallet_rpc KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t height; uint64_t spent; @@ -1628,6 +1740,7 @@ namespace wallet_rpc KV_SERIALIZE(unspent) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct uri_spec @@ -1649,11 +1762,12 @@ namespace wallet_rpc struct COMMAND_RPC_MAKE_URI { - struct request: public uri_spec + struct request_t: public uri_spec { }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string uri; @@ -1661,11 +1775,12 @@ namespace wallet_rpc KV_SERIALIZE(uri) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_PARSE_URI { - struct request + struct request_t { std::string uri; @@ -1673,8 +1788,9 @@ namespace wallet_rpc KV_SERIALIZE(uri) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uri_spec uri; std::vector<std::string> unknown_parameters; @@ -1684,11 +1800,12 @@ namespace wallet_rpc KV_SERIALIZE(unknown_parameters); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY { - struct request + struct request_t { std::string address; std::string payment_id; @@ -1700,8 +1817,9 @@ namespace wallet_rpc KV_SERIALIZE(description) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t index; @@ -1709,11 +1827,12 @@ namespace wallet_rpc KV_SERIALIZE(index); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY { - struct request + struct request_t { std::list<uint64_t> entries; @@ -1721,6 +1840,7 @@ namespace wallet_rpc KV_SERIALIZE(entries) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; struct entry { @@ -1737,7 +1857,7 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; - struct response + struct response_t { std::vector<entry> entries; @@ -1745,11 +1865,12 @@ namespace wallet_rpc KV_SERIALIZE(entries) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY { - struct request + struct request_t { uint64_t index; @@ -1757,32 +1878,36 @@ namespace wallet_rpc KV_SERIALIZE(index); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_RESCAN_SPENT { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_REFRESH { - struct request + struct request_t { uint64_t start_height; @@ -1790,8 +1915,9 @@ namespace wallet_rpc KV_SERIALIZE_OPT(start_height, (uint64_t) 0) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t blocks_fetched; bool received_money; @@ -1801,11 +1927,12 @@ namespace wallet_rpc KV_SERIALIZE(received_money); END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_START_MINING { - struct request + struct request_t { uint64_t threads_count; bool do_background_mining; @@ -1817,37 +1944,43 @@ namespace wallet_rpc KV_SERIALIZE(ignore_battery) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_STOP_MINING { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_LANGUAGES { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; - struct response + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t { std::vector<std::string> languages; @@ -1855,11 +1988,12 @@ namespace wallet_rpc KV_SERIALIZE(languages) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CREATE_WALLET { - struct request + struct request_t { std::string filename; std::string password; @@ -1871,16 +2005,19 @@ namespace wallet_rpc KV_SERIALIZE(language) END_KV_SERIALIZE_MAP() }; - struct response + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_OPEN_WALLET { - struct request + struct request_t { std::string filename; std::string password; @@ -1890,31 +2027,36 @@ namespace wallet_rpc KV_SERIALIZE(password) END_KV_SERIALIZE_MAP() }; - struct response + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CLOSE_WALLET { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_CHANGE_WALLET_PASSWORD { - struct request + struct request_t { std::string old_password; std::string new_password; @@ -1924,16 +2066,52 @@ namespace wallet_rpc KV_SERIALIZE(new_password) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_GENERATE_FROM_KEYS + { + struct request + { + uint64_t restore_height; + std::string filename; + std::string address; + std::string spendkey; + std::string viewkey; + std::string password; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(restore_height, (uint64_t)0) + KV_SERIALIZE(filename) + KV_SERIALIZE(address) + KV_SERIALIZE(spendkey) + KV_SERIALIZE(viewkey) + KV_SERIALIZE(password) + END_KV_SERIALIZE_MAP() + }; + struct response { + std::string address; + std::string info; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(info) END_KV_SERIALIZE_MAP() }; }; struct COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET { - struct request + struct request_t { uint64_t restore_height; std::string filename; @@ -1951,8 +2129,9 @@ namespace wallet_rpc KV_SERIALIZE(language) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string address; std::string seed; @@ -1966,17 +2145,19 @@ namespace wallet_rpc KV_SERIALIZE(was_deprecated) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_IS_MULTISIG { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { bool multisig; bool ready; @@ -1990,17 +2171,19 @@ namespace wallet_rpc KV_SERIALIZE(total) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_PREPARE_MULTISIG { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string multisig_info; @@ -2008,11 +2191,12 @@ namespace wallet_rpc KV_SERIALIZE(multisig_info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_MAKE_MULTISIG { - struct request + struct request_t { std::vector<std::string> multisig_info; uint32_t threshold; @@ -2024,8 +2208,9 @@ namespace wallet_rpc KV_SERIALIZE(password) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string address; std::string multisig_info; @@ -2035,17 +2220,19 @@ namespace wallet_rpc KV_SERIALIZE(multisig_info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_EXPORT_MULTISIG { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string info; @@ -2053,11 +2240,12 @@ namespace wallet_rpc KV_SERIALIZE(info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_IMPORT_MULTISIG { - struct request + struct request_t { std::vector<std::string> info; @@ -2065,8 +2253,9 @@ namespace wallet_rpc KV_SERIALIZE(info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint64_t n_outputs; @@ -2074,11 +2263,12 @@ namespace wallet_rpc KV_SERIALIZE(n_outputs) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_FINALIZE_MULTISIG { - struct request + struct request_t { std::string password; std::vector<std::string> multisig_info; @@ -2088,8 +2278,9 @@ namespace wallet_rpc KV_SERIALIZE(multisig_info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string address; @@ -2097,11 +2288,12 @@ namespace wallet_rpc KV_SERIALIZE(address) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_EXCHANGE_MULTISIG_KEYS { - struct request + struct request_t { std::string password; std::vector<std::string> multisig_info; @@ -2111,8 +2303,9 @@ namespace wallet_rpc KV_SERIALIZE(multisig_info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string address; std::string multisig_info; @@ -2122,11 +2315,12 @@ namespace wallet_rpc KV_SERIALIZE(multisig_info) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SIGN_MULTISIG { - struct request + struct request_t { std::string tx_data_hex; @@ -2134,8 +2328,9 @@ namespace wallet_rpc KV_SERIALIZE(tx_data_hex) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::string tx_data_hex; std::list<std::string> tx_hash_list; @@ -2145,11 +2340,12 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash_list) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_SUBMIT_MULTISIG { - struct request + struct request_t { std::string tx_data_hex; @@ -2157,8 +2353,9 @@ namespace wallet_rpc KV_SERIALIZE(tx_data_hex) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { std::list<std::string> tx_hash_list; @@ -2166,17 +2363,19 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash_list) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; }; struct COMMAND_RPC_GET_VERSION { - struct request + struct request_t { BEGIN_KV_SERIALIZE_MAP() END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<request_t> request; - struct response + struct response_t { uint32_t version; @@ -2184,6 +2383,42 @@ namespace wallet_rpc KV_SERIALIZE(version) END_KV_SERIALIZE_MAP() }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_VALIDATE_ADDRESS + { + struct request_t + { + std::string address; + bool any_net_type; + bool allow_openalias; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE_OPT(any_net_type, false) + KV_SERIALIZE_OPT(allow_openalias, false) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + bool valid; + bool integrated; + bool subaddress; + std::string nettype; + std::string openalias_address; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(valid) + KV_SERIALIZE(integrated) + KV_SERIALIZE(subaddress) + KV_SERIALIZE(nettype) + KV_SERIALIZE(openalias_address) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; }; } diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index 9b3a2847d..440a58a47 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // |