diff options
Diffstat (limited to 'src')
221 files changed, 21397 insertions, 7940 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b71c38cd..6ee7effdd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -145,3 +145,4 @@ if(PER_BLOCK_CHECKPOINT) endif() add_subdirectory(device) +add_subdirectory(device_trezor) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index e1b76ec1e..6c79120e8 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -224,7 +224,7 @@ struct Dbt_safe : public Dbt namespace cryptonote { -void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) +void BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -255,7 +255,7 @@ void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const if (res) throw0(DB_ERROR("Failed to add block blob to db transaction.")); - Dbt_copy<size_t> sz(block_size); + Dbt_copy<size_t> sz(block_weight); if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0)) throw0(DB_ERROR("Failed to add block size to db transaction.")); @@ -1213,6 +1213,11 @@ std::vector<std::string> BlockchainBDB::get_filenames() const return full_paths; } +bool BlockchainBDB::remove_data_file(const std::string& folder) +{ + return true; +} + std::string BlockchainBDB::get_db_name() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -1348,7 +1353,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainBDB::get_block_size(const uint64_t& height) const +size_t BlockchainBDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1856,7 +1861,7 @@ void BlockchainBDB::block_txn_abort() // TODO } -uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) +uint64_t BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1869,7 +1874,7 @@ uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, co uint64_t num_outputs = m_num_outputs; try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); m_write_txn = NULL; TIME_MEASURE_START(time1); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index cecbba28f..76d0a0517 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -244,6 +244,8 @@ public: virtual std::vector<std::string> get_filenames() const; + virtual bool remove_data_file(const std::string& folder); + virtual std::string get_db_name() const; virtual bool lock(); @@ -264,7 +266,7 @@ public: virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -301,7 +303,6 @@ public: virtual uint64_t get_indexing_base() const { return 1; } virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -317,7 +318,7 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -352,7 +353,7 @@ public: private: virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const crypto::hash& block_hash @@ -417,6 +418,7 @@ private: * @return the global index of the desired output */ uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); + output_data_t get_output_key(const uint64_t& global_index) const; void checkpoint_worker() const; void check_open() const; diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 88ac34255..be0ffeac3 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -87,8 +87,8 @@ const command_line::arg_descriptor<std::string> arg_db_type = { }; const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" -, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[nblocks_per_sync]." -, "fast:async:1000" +, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]." +, "fast:async:250000000bytes" }; const command_line::arg_descriptor<bool> arg_db_salvage = { "db-salvage" @@ -196,7 +196,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } uint64_t BlockchainDB::add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -218,13 +218,22 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to add the transactions time1 = epee::misc_utils::get_tick_count(); + + uint64_t num_rct_outs = 0; add_transaction(blk_hash, blk.miner_tx); + if (blk.miner_tx.version == 2) + num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; for (const transaction& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); + for (const auto &vout: tx.vout) + { + if (vout.amount == 0) + ++num_rct_outs; + } ++tx_i; } TIME_MEASURE_FINISH(time1); @@ -232,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); + add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 19ba32340..71c46d76b 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -137,7 +137,7 @@ struct txpool_tx_meta_t { crypto::hash max_used_block_id; crypto::hash last_failed_id; - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t max_used_block_height; uint64_t last_failed_height; @@ -148,6 +148,7 @@ struct txpool_tx_meta_t uint8_t relayed; uint8_t do_not_relay; uint8_t double_spend_seen: 1; + uint8_t bf_padding: 7; uint8_t padding[76]; // till 192 bytes }; @@ -357,15 +358,16 @@ private: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , uint64_t num_rct_outs , const crypto::hash& blk_hash ) = 0; @@ -374,7 +376,7 @@ private: * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -542,7 +544,7 @@ public: /** * @brief An empty constructor. */ - BlockchainDB(): m_open(false) { } + BlockchainDB(): m_hardfork(NULL), m_open(false) { } /** * @brief An empty destructor. @@ -654,6 +656,20 @@ public: */ virtual std::vector<std::string> get_filenames() const = 0; + /** + * @brief remove file(s) storing the database + * + * This function is for resetting the database (for core tests, functional tests, etc). + * The function reset() is not usable because it needs to open the database file first + * which can fail if the existing database file is in an incompatible format. + * As such, this function needs to be called before calling open(). + * + * @param folder The path of the folder containing the database file(s) which must not end with slash '/'. + * + * @return true if the operation is succesfull + */ + virtual bool remove_data_file(const std::string& folder) const = 0; + // return the name of the folder the db's file(s) should reside in /** * @brief gets the name of the folder the BlockchainDB's file(s) should be in @@ -773,7 +789,7 @@ public: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the size of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block @@ -781,7 +797,7 @@ public: * @return the height of the chain post-addition */ virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -891,6 +907,20 @@ public: virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; /** + * @brief fetch a block's cumulative number of rct outputs + * + * The subclass should return the numer of rct outputs in the blockchain + * up to the block with the given height (inclusive). + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param height the height requested + * + * @return the cumulative number of rct outputs + */ + virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const = 0; + + /** * @brief fetch the top block's timestamp * * The subclass should return the timestamp of the most recent block. @@ -900,18 +930,18 @@ public: virtual uint64_t get_top_block_timestamp() const = 0; /** - * @brief fetch a block's size + * @brief fetch a block's weight * - * The subclass should return the size of the block with the + * The subclass should return the weight of the block with the * given height. * * If the block does not exist, the subclass should throw BLOCK_DNE * * @param height the height requested * - * @return the size + * @return the weight */ - virtual size_t get_block_size(const uint64_t& height) const = 0; + virtual size_t get_block_weight(const uint64_t& height) const = 0; /** * @brief fetch a block's cumulative difficulty @@ -1232,23 +1262,6 @@ public: virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0; /** - * @brief get some of an output's data - * - * The subclass should return the public key, unlock time, and block height - * for the output with the given global index, collected in a struct. - * - * If the output cannot be found, the subclass should throw OUTPUT_DNE. - * - * If any of these parts cannot be found, but some are, the subclass - * should throw DB_ERROR with a message stating as much. - * - * @param global_index the output's index (global) - * - * @return the requested output data - */ - virtual output_data_t get_output_key(const uint64_t& global_index) const = 0; - - /** * @brief gets an output's tx hash and index * * The subclass should return the hash of the transaction which created the @@ -1535,6 +1548,13 @@ public: */ virtual bool is_read_only() const = 0; + /** + * @brief get disk space requirements + * + * @return the size required + */ + virtual uint64_t get_database_size() const = 0; + // TODO: this should perhaps be (or call) a series of functions which // progressively update through version updates /** diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index fe31321f3..84a083c26 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -35,6 +35,7 @@ #include <random> #include "string_tools.h" +#include "file_io_utils.h" #include "common/util.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "crypto/crypto.h" @@ -53,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 2 +#define VERSION 3 namespace { @@ -250,14 +251,25 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { +typedef struct mdb_block_info_old +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; +} mdb_block_info_old; + typedef struct mdb_block_info { uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat + uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; + uint64_t bi_cum_rct; } mdb_block_info; typedef struct blk_height { @@ -533,8 +545,8 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const LOG_PRINT_L1("Space used: " << size_used); LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); LOG_PRINT_L1("Size threshold: " << threshold_size); - float resize_percent_old = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old); + float resize_percent = RESIZE_PERCENT; + LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { @@ -547,10 +559,6 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const return false; } - std::mt19937 engine(std::random_device{}()); - std::uniform_real_distribution<double> fdis(0.6, 0.9); - double resize_percent = fdis(engine); - if ((double)size_used / mei.me_mapsize > resize_percent) { LOG_PRINT_L1("Threshold met (percent-based)"); @@ -641,16 +649,19 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin { MDB_txn *rtxn; mdb_txn_cursors *rcurs; - block_rtxn_start(&rtxn, &rcurs); + bool my_rtxn = block_rtxn_start(&rtxn, &rcurs); for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) { - uint32_t block_size = get_block_size(block_num); - total_block_size += block_size; + // we have access to block weight, which will be greater or equal to block size, + // so use this as a proxy. If it's too much off, we might have to check actual size, + // which involves reading more data, so is not really wanted + size_t block_weight = get_block_weight(block_num); + total_block_size += block_weight; // Track number of blocks being totalled here instead of assuming, in case // some blocks were to be skipped for being outliers. ++num_blocks_used; } - block_rtxn_stop(); + if (my_rtxn) block_rtxn_stop(); avg_block_size = total_block_size / num_blocks_used; MDEBUG("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size); } @@ -666,8 +677,8 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - const crypto::hash& blk_hash) +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, + uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -712,9 +723,19 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_height = m_height; bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; - bi.bi_size = block_size; + bi.bi_weight = block_weight; bi.bi_diff = cumulative_difficulty; bi.bi_hash = blk_hash; + bi.bi_cum_rct = num_rct_outs; + if (blk.major_version >= 4) + { + uint64_t last_height = m_height-1; + MDB_val_set(h, last_height); + if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) + throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str())); + const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; + bi.bi_cum_rct += bi_prev->bi_cum_rct; + } MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); @@ -725,7 +746,9 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - m_cum_size += block_size; + // we use weight as a proxy for size, since we don't have size but weight is >= size + // and often actually equal + m_cum_size += block_weight; m_cum_count++; } @@ -759,8 +782,6 @@ void BlockchainLMDB::remove_block() if ((result = mdb_cursor_del(m_cur_block_heights, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET))) - throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str())); if ((result = mdb_cursor_del(m_cur_blocks, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); @@ -1124,7 +1145,10 @@ BlockchainLMDB::~BlockchainLMDB() // batch transaction shouldn't be active at this point. If it is, consider it aborted. if (m_batch_active) - batch_abort(); + { + try { batch_abort(); } + catch (...) { /* ignore */ } + } if (m_open) close(); } @@ -1143,6 +1167,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB() m_cum_size = 0; m_cum_count = 0; + // reset may also need changing when initialize things here + m_hardfork = nullptr; } @@ -1178,8 +1204,12 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) throw DB_ERROR("Database could not be opened"); } - if (tools::is_hdd(filename.c_str())) - MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + boost::optional<bool> is_hdd_result = tools::is_hdd(filename.c_str()); + if (is_hdd_result) + { + if (is_hdd_result.value()) + MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use an SSD if possible"); + } m_folder = filename; @@ -1306,20 +1336,21 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) auto get_result = mdb_get(txn, m_properties, &k, &v); if(get_result == MDB_SUCCESS) { - if (*(const uint32_t*)v.mv_data > VERSION) + const uint32_t db_version = *(const uint32_t*)v.mv_data; + if (db_version > VERSION) { - MWARNING("Existing lmdb database was made by a later version. We don't know how it will change yet."); + MWARNING("Existing lmdb database was made by a later version (" << db_version << "). We don't know how it will change yet."); compatible = false; } #if VERSION > 0 - else if (*(const uint32_t*)v.mv_data < VERSION) + else if (db_version < VERSION) { // Note that there was a schema change within version 0 as well. // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 // We don't handle the old format previous to that commit. txn.commit(); m_open = true; - migrate(*(const uint32_t *)v.mv_data); + migrate(db_version); return; } #endif @@ -1390,6 +1421,9 @@ void BlockchainLMDB::sync() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (is_read_only()) + return; + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part // MDB_NOMETASYNC. Force flush to be synchronous. if (auto result = mdb_env_sync(m_env, true)) @@ -1468,6 +1502,21 @@ std::vector<std::string> BlockchainLMDB::get_filenames() const return filenames; } +bool BlockchainLMDB::remove_data_file(const std::string& folder) const +{ + const std::string filename = folder + "/data.mdb"; + try + { + boost::filesystem::remove(filename); + } + catch (const std::exception &e) + { + MERROR("Failed to remove " << filename << ": " << e.what()); + return false; + } + return true; +} + std::string BlockchainLMDB::get_db_name() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1907,6 +1956,68 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const return ret; } +std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<uint64_t> res; + int result; + + if (heights.empty()) + return {}; + res.reserve(heights.size()); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_stat db_stats; + if ((result = mdb_stat(m_txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + for (size_t i = 0; i < heights.size(); ++i) + if (heights[i] >= db_stats.ms_entries) + throw0(BLOCK_DNE(std::string("Attempt to get rct distribution from height " + std::to_string(heights[i]) + " failed -- block size not in db").c_str())); + + MDB_val v; + + uint64_t prev_height = heights[0]; + uint64_t range_begin = 0, range_end = 0; + for (uint64_t height: heights) + { + if (height >= range_begin && height < range_end) + { + // nohting to do + } + else + { + if (height == prev_height + 1) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE); + range_begin = ((const mdb_block_info*)v.mv_data)->bi_height; + range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please + if (height < range_begin || height >= range_end) + throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multuple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str())); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + range_begin = height; + range_end = range_begin + 1; + } + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); + } + const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin); + res.push_back(bi->bi_cum_rct); + prev_height = height; + } + + TXN_POSTFIX_RDONLY(); + return res; +} + uint64_t BlockchainLMDB::get_top_block_timestamp() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1922,7 +2033,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1940,7 +2051,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; - size_t ret = bi->bi_size; + size_t ret = bi->bi_weight; TXN_POSTFIX_RDONLY(); return ret; } @@ -2376,55 +2487,6 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -// This is a lot harder now that we've removed the output_keys index -output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - check_open(); - TXN_PREFIX_RDONLY(); - RCURSOR(output_txs); - RCURSOR(tx_indices); - - output_data_t od; - MDB_val_set(v, global_index); - auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - outtx *ot = (outtx *)v.mv_data; - - MDB_val_set(val_h, ot->tx_hash); - get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); - if (get_result) - throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(ot->tx_hash) + ": ", get_result).c_str())); - - txindex *tip = (txindex *)val_h.mv_data; - MDB_val_set(val_tx_id, tip->data.tx_id); - MDB_val result; - get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); - - blobdata bd; - bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); - - transaction tx; - if (!parse_and_validate_tx_base_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - - const tx_out tx_output = tx.vout[ot->local_index]; - od.unlock_time = tip->data.unlock_time; - od.height = tip->data.block_id; - od.pubkey = boost::get<txout_to_key>(tx_output.target).key; - - TXN_POSTFIX_RDONLY(); - return od; -} - output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2907,10 +2969,10 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("BlockchainLMDB::" << __func__); if ((batch_transactions) && (m_batch_transactions)) { - LOG_PRINT_L0("WARNING: batch transaction mode already enabled, but asked to enable batch mode"); + MINFO("batch transaction mode already enabled, but asked to enable batch mode"); } m_batch_transactions = batch_transactions; - LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); + MINFO("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } // return true if we started the txn, false if already started @@ -3050,7 +3112,7 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3069,7 +3131,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -3327,6 +3389,7 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig MDB_val_set(k, amount); MDB_val v; MDB_cursor_op op = MDB_SET; + base = 0; while (1) { int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); @@ -3345,6 +3408,11 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig break; } + distribution[0] += base; + for (size_t n = 1; n < distribution.size(); ++n) + distribution[n] += distribution[n - 1]; + base = 0; + TXN_POSTFIX_RDONLY(); return true; @@ -3422,6 +3490,16 @@ bool BlockchainLMDB::is_read_only() const return false; } +uint64_t BlockchainLMDB::get_database_size() const +{ + uint64_t size = 0; + boost::filesystem::path datafile(m_folder); + datafile /= CRYPTONOTE_BLOCKCHAINDATA_FILENAME; + if (!epee::file_io_utils::get_file_size(datafile.string(), size)) + size = 0; + return size; +} + void BlockchainLMDB::fixup() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3429,7 +3507,17 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } -#define RENAME_DB(name) \ +#define RENAME_DB(name) do { \ + char n2[] = name; \ + MDB_dbi tdbi; \ + n2[sizeof(n2)-2]--; \ + /* play some games to put (name) on a writable page */ \ + result = mdb_dbi_open(txn, n2, MDB_CREATE, &tdbi); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to create " + std::string(n2) + ": ", result).c_str())); \ + result = mdb_drop(txn, tdbi, 1); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to delete " + std::string(n2) + ": ", result).c_str())); \ k.mv_data = (void *)name; \ k.mv_size = sizeof(name)-1; \ result = mdb_cursor_open(txn, 1, &c_cur); \ @@ -3439,7 +3527,7 @@ void BlockchainLMDB::fixup() if (result) \ throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \ ptr = (char *)k.mv_data; \ - ptr[sizeof(name)-2] = 's' + ptr[sizeof(name)-2]++; } while(0) #define LOGIF(y) if (ELPP->vRegistry()->allowed(y, "global")) @@ -3518,7 +3606,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_heights: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_block_heights, &ms); + result = mdb_stat(txn, m_block_heights, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query block_heights table: ", result).c_str())); i = ms.ms_entries; } } @@ -3579,7 +3669,7 @@ void BlockchainLMDB::migrate_0_1() break; } MDB_dbi diffs, hashes, sizes, timestamps; - mdb_block_info bi; + mdb_block_info_old bi; MDB_val_set(nv, bi); lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); @@ -3621,7 +3711,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_timestamps: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_block_info, &ms); + result = mdb_stat(txn, m_block_info, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query block_info table: ", result).c_str())); i = ms.ms_entries; } } @@ -3644,9 +3736,9 @@ void BlockchainLMDB::migrate_0_1() if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); if (v.mv_size == sizeof(uint32_t)) - bi.bi_size = *(uint32_t *)v.mv_data; + bi.bi_weight = *(uint32_t *)v.mv_data; else - bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + bi.bi_weight = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); @@ -3741,7 +3833,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for spent_keys: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_hf_versions, &ms); + result = mdb_stat(txn, m_hf_versions, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query hf_versions table: ", result).c_str())); i = ms.ms_entries; } } @@ -3896,7 +3990,9 @@ void BlockchainLMDB::migrate_0_1() throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); if (!i) { MDB_stat ms; - mdb_stat(txn, m_txs, &ms); + result = mdb_stat(txn, m_txs, &ms); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query txs table: ", result).c_str())); i = ms.ms_entries; if (i) { MDB_val_set(pk, "txblk"); @@ -4119,6 +4215,141 @@ void BlockchainLMDB::migrate_1_2() txn.commit(); } +void BlockchainLMDB::migrate_2_3() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 2 to 3 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + MDEBUG("enumerating rct outputs..."); + std::vector<uint64_t> distribution(blockchain_height, 0); + bool r = for_all_outputs(0, [&](uint64_t height) { + if (height >= blockchain_height) + { + MERROR("Output found claiming height >= blockchain height"); + return false; + } + distribution[height]++; + return true; + }); + if (!r) + throw0(DB_ERROR("Failed to build rct output distribution")); + for (size_t i = 1; i < distribution.size(); ++i) + distribution[i] += distribution[i - 1]; + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_old *bi_old = (const mdb_block_info_old*)v.mv_data; + mdb_block_info bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + if (bi_old->bi_height >= distribution.size()) + throw0(DB_ERROR("Bad height in block_info record")); + bi.bi_cum_rct = distribution[bi_old->bi_height]; + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 3; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_copy<const char *> vk("version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { switch(oldversion) { @@ -4126,6 +4357,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_0_1(); /* FALLTHRU */ case 1: migrate_1_2(); /* FALLTHRU */ + case 2: + migrate_2_3(); /* FALLTHRU */ default: ; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index cc1b06ca0..e1f748ed8 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -166,7 +166,7 @@ struct mdb_txn_safe class BlockchainLMDB : public BlockchainDB { public: - BlockchainLMDB(bool batch_transactions=false); + BlockchainLMDB(bool batch_transactions=true); ~BlockchainLMDB(); virtual void open(const std::string& filename, const int mdb_flags=0); @@ -181,6 +181,8 @@ public: virtual std::vector<std::string> get_filenames() const; + virtual bool remove_data_file(const std::string& folder) const; + virtual std::string get_db_name() const; virtual bool lock(); @@ -197,11 +199,13 @@ public: virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const; + virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const; + virtual uint64_t get_block_timestamp(const uint64_t& height) const; virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -239,7 +243,6 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -270,7 +273,7 @@ public: virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -314,9 +317,10 @@ private: uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const; virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , uint64_t num_rct_outs , const crypto::hash& block_hash ); @@ -375,6 +379,8 @@ private: virtual bool is_read_only() const; + virtual uint64_t get_database_size() const; + // fix up anything that may be wrong due to past bugs virtual void fixup(); @@ -387,6 +393,9 @@ private: // migrate from DB version 1 to 2 void migrate_1_2(); + // migrate from DB version 2 to 3 + void migrate_2_3(); + void cleanup_batch(); private: @@ -440,7 +449,7 @@ private: #endif #endif - constexpr static float RESIZE_PERCENT = 0.8f; + constexpr static float RESIZE_PERCENT = 0.9f; }; } // namespace cryptonote diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 338ec3e4b..c9ad1cebe 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -26,16 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(blocksdat "") -if(PER_BLOCK_CHECKPOINT) - if(APPLE) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - else() - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - endif() - set(blocksdat "blocksdat.o") -endif() - set(blockchain_import_sources blockchain_import.cpp bootstrap_file.cpp @@ -91,10 +81,39 @@ monero_private_headers(blockchain_usage +set(blockchain_ancestry_sources + blockchain_ancestry.cpp + ) + +set(blockchain_ancestry_private_headers) + +monero_private_headers(blockchain_ancestry + ${blockchain_ancestry_private_headers}) + + + +set(blockchain_depth_sources + blockchain_depth.cpp + ) + +set(blockchain_depth_private_headers) + +monero_private_headers(blockchain_depth + ${blockchain_depth_private_headers}) + +set(blockchain_stats_sources + blockchain_stats.cpp + ) + +set(blockchain_stats_private_headers) + +monero_private_headers(blockchain_stats + ${blockchain_stats_private_headers}) + + monero_add_executable(blockchain_import ${blockchain_import_sources} - ${blockchain_import_private_headers} - ${blocksdat}) + ${blockchain_import_private_headers}) target_link_libraries(blockchain_import PRIVATE @@ -106,7 +125,8 @@ target_link_libraries(blockchain_import ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${Blocks}) if(ARCH_WIDTH) target_compile_definitions(blockchain_import @@ -158,7 +178,7 @@ target_link_libraries(blockchain_blackball set_property(TARGET blockchain_blackball PROPERTY - OUTPUT_NAME "monero-blockchain-blackball") + OUTPUT_NAME "monero-blockchain-mark-spent-outputs") install(TARGETS blockchain_blackball DESTINATION bin) @@ -183,3 +203,65 @@ set_property(TARGET blockchain_usage OUTPUT_NAME "monero-blockchain-usage") install(TARGETS blockchain_usage DESTINATION bin) +monero_add_executable(blockchain_ancestry + ${blockchain_ancestry_sources} + ${blockchain_ancestry_private_headers}) + +target_link_libraries(blockchain_ancestry + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_ancestry + PROPERTY + OUTPUT_NAME "monero-blockchain-ancestry") +install(TARGETS blockchain_ancestry DESTINATION bin) + +monero_add_executable(blockchain_depth + ${blockchain_depth_sources} + ${blockchain_depth_private_headers}) + +target_link_libraries(blockchain_depth + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_depth + PROPERTY + OUTPUT_NAME "monero-blockchain-depth") +install(TARGETS blockchain_depth DESTINATION bin) + +monero_add_executable(blockchain_stats + ${blockchain_stats_sources} + ${blockchain_stats_private_headers}) + +target_link_libraries(blockchain_stats + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_stats + PROPERTY + OUTPUT_NAME "monero-blockchain-stats") +install(TARGETS blockchain_stats DESTINATION bin) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp new file mode 100644 index 000000000..e01a8892c --- /dev/null +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -0,0 +1,768 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <unordered_map> +#include <unordered_set> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/archive/portable_binary_oarchive.hpp> +#include "common/unordered_containers_boost_serialization.h" +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +struct ancestor +{ + uint64_t amount; + uint64_t offset; + + bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & amount; + a & offset; + } +}; +BOOST_CLASS_VERSION(ancestor, 0) + +namespace std +{ + template<> struct hash<ancestor> + { + size_t operator()(const ancestor &a) const + { + return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't + } + }; +} + +struct tx_data_t +{ + std::vector<std::pair<uint64_t, std::vector<uint64_t>>> vin; + std::vector<crypto::public_key> vout; + bool coinbase; + + tx_data_t(): coinbase(false) {} + tx_data_t(const cryptonote::transaction &tx) + { + coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!coinbase) + { + vin.reserve(tx.vin.size()); + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets))); + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vin type"); + } + } + } + vout.reserve(tx.vout.size()); + for (size_t out = 0; out < tx.vout.size(); ++out) + { + if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx.vout[out].target); + vout.push_back(txout.key); + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vout type"); + } + } + } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & coinbase; + a & vin; + a & vout; + } +}; + +struct ancestry_state_t +{ + uint64_t height; + std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry; + std::unordered_map<ancestor, crypto::hash> output_cache; + std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; + std::vector<cryptonote::block> block_cache; + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & height; + a & ancestry; + a & output_cache; + if (ver < 1) + { + std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache; + a & old_tx_cache; + for (const auto i: old_tx_cache) + tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); + } + else + { + a & tx_cache; + } + if (ver < 2) + { + std::unordered_map<uint64_t, cryptonote::block> old_block_cache; + a & old_block_cache; + block_cache.resize(old_block_cache.size()); + for (const auto i: old_block_cache) + block_cache[i.first] = i.second; + } + else + { + a & block_cache; + } + } +}; +BOOST_CLASS_VERSION(ancestry_state_t, 2) + +static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset) +{ + std::pair<std::unordered_map<ancestor, unsigned int>::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); + if (!p.second) + { + ++p.first->second; + } +} + +static size_t get_full_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + size_t count = 0; + for (const auto &i: ancestry) + count += i.second; + return count; +} + +static size_t get_deduplicated_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + return ancestry.size(); +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const std::unordered_set<ancestor> &ancestors) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors)); + if (!p.second) + { + for (const auto &e: ancestors) + p.first->second.insert(e); + } +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>())); + p.first->second.insert(new_ancestor); +} + +static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid) +{ + std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid); + if (i == ancestry.end()) + { + //MERROR("txid ancestry not found: " << txid); + //throw std::runtime_error("txid ancestry not found"); + return std::unordered_set<ancestor>(); + } + return i->second; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_cache_outputs); + command_line::add_arg(desc_cmd_sett, arg_cache_txes); + command_line::add_arg(desc_cmd_sett, arg_cache_blocks); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_sett, arg_show_cache_stats); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_all = command_line::get_arg(vm, arg_all); + bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); + + if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + { + std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + + // forward method + if (opt_all) + { + uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; + ancestry_state_t state; + + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try + { + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); + } + state_data_in.close(); + } + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + MINFO("Starting from height " << state.height); + const uint64_t db_height = db->height(); + state.block_cache.reserve(db_height); + for (uint64_t h = state.height; h < db_height; ++h) + { + size_t block_ancestry_size = 0; + const cryptonote::blobdata bd = db->get_block_blob_from_height(h); + ++total_blocks; + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(h + 1); + state.block_cache[h] = b; + } + std::vector<crypto::hash> txids; + txids.reserve(1 + b.tx_hashes.size()); + if (opt_include_coinbase) + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + for (const auto &h: b.tx_hashes) + txids.push_back(h); + for (const crypto::hash &txid: txids) + { + printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); + fflush(stdout); + ::tx_data_t tx_data; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + } + if (tx_data.coinbase) + { + add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>()); + } + else + { + for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) + { + if (1) + { + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + cryptonote::block b; + ++total_blocks; + if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[od.height]; + } + else + { + cryptonote::blobdata bd = db->get_block_blob_from_height(od.height); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(od.height + 1); + state.block_cache[od.height] = b; + } + } + // find the tx which created this output + bool found = false; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + ++total_outputs; + if (i != state.output_cache.end()) + { + ++cached_outputs; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); + found = true; + } + else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + ::tx_data_t tx_data2; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data2 = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + tx_data2 = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); + } + for (size_t out = 0; out < tx_data2.vout.size(); ++out) + { + if (tx_data2.vout[out] == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + break; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + } + } + const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); + block_ancestry_size += ancestry_size; + MINFO(txid << ": " << ancestry_size); + } + if (!txids.empty()) + { + std::string stats_msg; + if (opt_show_cache_stats) + stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) + + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " + + std::to_string(cached_outputs*100./total_outputs); + MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); + } + state.height = h; + if (stop_requested) + break; + } + + LOG_PRINT_L0("Saving state data to " << state_file_path); + std::ofstream state_data_out; + state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (!state_data_out.fail()) + { + try + { + boost::archive::portable_binary_oarchive a(state_data_out); + a << state; + } + catch (const std::exception &e) + { + MERROR("Failed to save state data to " << state_file_path); + } + state_data_out.close(); + } + + goto done; + } + + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + for (const crypto::hash &start_txid: start_txids) + { + LOG_PRINT_L0("Checking ancestry for txid " << start_txid); + + std::unordered_map<ancestor, unsigned int> ancestry; + + std::list<crypto::hash> txids; + txids.push_back(start_txid); + while (!txids.empty()) + { + const crypto::hash txid = txids.front(); + txids.pop_front(); + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (coinbase) + continue; + + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + add_ancestor(ancestry, amount, offset); + const output_data_t od = db->get_output_key(amount, offset); + bd = db->get_block_blob_from_height(od.height); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + + MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry)); + for (const auto &i: ancestry) + { + MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second); + } + } + +done: + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 95eb2f73d..d2ce5cf76 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -50,64 +50,86 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; +static const char zerokey[8] = {0}; +static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; + +static uint64_t records_per_sync = 200; +static uint64_t db_flags = 0; +static MDB_dbi dbi_relative_rings; +static MDB_dbi dbi_outputs; +static MDB_dbi dbi_processed_txidx; +static MDB_dbi dbi_spent; +static MDB_dbi dbi_ring_instances; +static MDB_dbi dbi_stats; +static MDB_env *env = NULL; + struct output_data { uint64_t amount; - uint64_t index; - output_data(): amount(0), index(0) {} - output_data(uint64_t a, uint64_t i): amount(a), index(i) {} - bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) - { - a & amount; - a & index; - } + uint64_t offset; + output_data(): amount(0), offset(0) {} + output_data(uint64_t a, uint64_t i): amount(a), offset(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.offset == offset; } }; -BOOST_CLASS_VERSION(output_data, 0) -namespace std +// +// relative_rings: key_image -> vector<uint64_t> +// outputs: 128 bits -> set of key images +// processed_txidx: string -> uint64_t +// spent: amount -> offset +// ring_instances: vector<uint64_t> -> uint64_t +// stats: string -> arbitrary +// + +static bool parse_db_sync_mode(std::string db_sync_mode) { - template<> struct hash<output_data> + std::vector<std::string> options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) { - size_t operator()(const output_data &od) const + if(options[0] == "safe") { - const uint64_t data[2] = {od.amount, od.index}; - crypto::hash h; - crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + safemode = true; + db_flags = DBF_SAFE; } - }; - template<> struct hash<std::vector<uint64_t>> - { - size_t operator()(const std::vector<uint64_t> &v) const + else if(options[0] == "fast") { - crypto::hash h; - crypto::cn_fast_hash(v.data(), v.size() * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + db_flags = DBF_FAST; } - }; -} - -struct blackball_state_t -{ - std::unordered_map<crypto::key_image, std::vector<uint64_t>> relative_rings; - std::unordered_map<output_data, std::unordered_set<crypto::key_image>> outputs; - std::unordered_map<std::string, uint64_t> processed_heights; - std::unordered_set<output_data> spent; - std::unordered_map<std::vector<uint64_t>, size_t> ring_instances; + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + db_flags = DEFAULT_FLAGS; + } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + if(options.size() >= 2 && !safemode) { - a & relative_rings; - a & outputs; - a & processed_heights; - a & spent; - if (ver < 1) - return; - a & ring_instances; + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr == '\0') + records_per_sync = bps; } -}; -BOOST_CLASS_VERSION(blackball_state_t, 1) + + return true; +} static std::string get_default_db_path() { @@ -118,7 +140,195 @@ static std::string get_default_db_path() return dir.string(); } -static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, const std::function<bool(const cryptonote::transaction_prefix&)> &f) +static std::string get_cache_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + const uint32_t *va = (const uint32_t*) a->mv_data; + const uint32_t *vb = (const uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} + +static int compare_double64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + if (va == vb) + { + const uint64_t va = ((const uint64_t*) a->mv_data)[1]; + const uint64_t vb = ((const uint64_t*) b->mv_data)[1]; + return va < vb ? -1 : va > vb; + } + return va < vb ? -1 : va > vb; +} + +static int resize_env(const char *db_path) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + size_t needed = 1000ul * 1024 * 1024; // at least 1000 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static void init(std::string cache_filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + MINFO("Creating spent output cache in " << cache_filename); + + tools::create_directories_if_necessary(cache_filename); + + int flags = 0; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + dbr = mdb_env_create(&env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 6); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_cache_filename(cache_filename); + dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "relative_rings", MDB_CREATE | MDB_INTEGERKEY, &dbi_relative_rings); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_relative_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, "outputs", MDB_CREATE | MDB_INTEGERKEY, &dbi_outputs); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_outputs, compare_double64); + + dbr = mdb_dbi_open(txn, "processed_txidx", MDB_CREATE, &dbi_processed_txidx); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_spent, compare_uint64); + + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +static void close() +{ + if (env) + { + mdb_dbi_close(env, dbi_relative_rings); + mdb_dbi_close(env, dbi_outputs); + mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_spent); + mdb_dbi_close(env, dbi_ring_instances); + mdb_dbi_close(env, dbi_stats); + mdb_env_close(env); + env = NULL; + } +} + +static std::string compress_ring(const std::vector<uint64_t> &ring, std::string s = "") +{ + const size_t sz = s.size(); + s.resize(s.size() + 12 * ring.size()); + char *ptr = (char*)s.data() + sz; + for (uint64_t out: ring) + tools::write_varint(ptr, out); + if (ptr > s.data() + sz + 12 * ring.size()) + throw std::runtime_error("varint output overflow"); + s.resize(ptr - s.data()); + return s; +} + +static std::string compress_ring(uint64_t amount, const std::vector<uint64_t> &ring) +{ + char s[12], *ptr = s; + tools::write_varint(ptr, amount); + if (ptr > s + sizeof(s)) + throw std::runtime_error("varint output overflow"); + return compress_ring(ring, std::string(s, ptr-s)); +} + +static std::vector<uint64_t> decompress_ring(const std::string &s) +{ + std::vector<uint64_t> ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + +static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, uint64_t &n_txes, const std::function<bool(const cryptonote::transaction_prefix&)> &f) { MDB_env *env; MDB_dbi dbi; @@ -126,6 +336,8 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id MDB_cursor *cur; int dbr; bool tx_active = false; + MDB_val k; + MDB_val v; dbr = mdb_env_create(&env); if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); @@ -136,7 +348,7 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open rings database file '" + actual_filename + "': " + std::string(mdb_strerror(dbr))); - dbr = mdb_txn_begin(env, NULL, 0, &txn); + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; @@ -147,9 +359,11 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn, dbi, &cur); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn, dbi, &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; - MDB_val k; - MDB_val v; bool fret = true; k.mv_size = sizeof(uint64_t); @@ -187,13 +401,86 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id } mdb_cursor_close(cur); - mdb_txn_commit(txn); + dbr = mdb_txn_commit(txn); + if (dbr) throw std::runtime_error("Failed to commit db transaction: " + std::string(mdb_strerror(dbr))); tx_active = false; mdb_dbi_close(env, dbi); mdb_env_close(env); return fret; } +static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) +{ + MDB_env *env[2]; + MDB_dbi dbi[2]; + MDB_txn *txn[2]; + MDB_cursor *cur[2]; + int dbr; + bool tx_active[2] = { false, false }; + uint64_t n_txes[2]; + MDB_val k; + MDB_val v[2]; + + epee::misc_utils::auto_scope_leave_caller txn_dtor[2]; + for (int i = 0; i < 2; ++i) + { + dbr = mdb_env_create(&env[i]); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env[i], 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = i ? second_filename : first_filename; + dbr = mdb_env_open(env[i], actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env[i], NULL, MDB_RDONLY, &txn[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + txn_dtor[i] = epee::misc_utils::create_scope_leave_handler([&, i](){if (tx_active[i]) mdb_txn_abort(txn[i]);}); + tx_active[i] = true; + + dbr = mdb_dbi_open(txn[i], "txs_pruned", MDB_INTEGERKEY, &dbi[i]); + if (dbr) + dbr = mdb_dbi_open(txn[i], "txs", MDB_INTEGERKEY, &dbi[i]); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn[i], dbi[i], &cur[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn[i], dbi[i], &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes[i] = stat.ms_entries; + } + + if (n_txes[0] == 0 || n_txes[1] == 0) + throw std::runtime_error("No transaction in the database"); + uint64_t lo = 0, hi = std::min(n_txes[0], n_txes[1]) - 1; + while (lo <= hi) + { + uint64_t mid = (lo + hi) / 2; + + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)∣ + dbr = mdb_cursor_get(cur[0], &k, &v[0], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cur[1], &k, &v[1], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + if (v[0].mv_size == v[1].mv_size && !memcmp(v[0].mv_data, v[1].mv_data, v[0].mv_size)) + lo = mid + 1; + else + hi = mid - 1; + } + + for (int i = 0; i < 2; ++i) + { + mdb_cursor_close(cur[i]); + dbr = mdb_txn_commit(txn[i]); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + tx_active[i] = false; + mdb_dbi_close(env[i], dbi[i]); + mdb_env_close(env[i]); + } + return hi; +} + static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) { std::vector<uint64_t> c; @@ -212,6 +499,506 @@ static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) return c; } +static uint64_t get_num_spent_outputs() +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0, tmp; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + dbr = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT_NODUP; + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first/next spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &tmp); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + count += tmp; + } + + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return count; +} + +static bool add_spent_output(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_put(cur, &k, &v, MDB_NODUPDATA); + if (dbr == MDB_KEYEXIST) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); + return true; +} + +static bool is_output_spent(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); + bool spent = dbr == 0; + return spent; +} + +static std::vector<output_data> get_spent_outputs(MDB_txn *txn) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &count); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + } + std::vector<output_data> outs; + outs.reserve(count); + while (1) + { + outs.push_back({*(const uint64_t*)k.mv_data, *(const uint64_t*)v.mv_data}); + dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT); + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get next spent output: " + std::string(mdb_strerror(dbr))); + } + mdb_cursor_close(cur); + return outs; +} + +static uint64_t get_processed_txidx(const std::string &name) +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + uint64_t height = 0; + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + dbr = mdb_get(txn, dbi_processed_txidx, &k, &v); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get processed height: " + std::string(mdb_strerror(dbr))); + height = *(const uint64_t*)v.mv_data; + } + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return height; +} + +static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t height) +{ + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + v.mv_data = (void*)&height; + v.mv_size = sizeof(height); + int dbr = mdb_put(txn, dbi_processed_txidx, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set processed height: " + std::string(mdb_strerror(dbr))); +} + +static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector<uint64_t> &ring) +{ + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + int dbr = mdb_get(txn, dbi_relative_rings, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get relative ring: " + std::string(mdb_strerror(dbr))); + ring = decompress_ring(std::string((const char*)v.mv_data, v.mv_size)); + return true; +} + +static void set_relative_ring(MDB_txn *txn, const crypto::key_image &ki, const std::vector<uint64_t> &ring) +{ + const std::string sring = compress_ring(ring); + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + v.mv_data = (void*)sring.c_str(); + v.mv_size = sring.size(); + int dbr = mdb_put(txn, dbi_relative_rings, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set relative ring: " + std::string(mdb_strerror(dbr))); +} + +static std::string keep_under_511(const std::string &s) +{ + if (s.size() <= 511) + return s; + crypto::hash hash; + crypto::cn_fast_hash(s.data(), s.size(), hash); + return std::string((const char*)&hash, 32); +} + +static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + if (dbr == MDB_NOTFOUND) + return 0; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + return *(const uint64_t*)v.mv_data; +} + +static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + uint64_t instances = get_ring_instances(txn, amount, ring); + if (ring.size() > 11) + return instances; + + uint64_t extra = 0; + std::vector<uint64_t> subset; + subset.reserve(ring.size()); + for (uint64_t mask = 1; mask < (((uint64_t)1) << ring.size()) - 1; ++mask) + { + subset.resize(0); + for (size_t i = 0; i < ring.size(); ++i) + if ((mask >> i) & 1) + subset.push_back(ring[i]); + extra += get_ring_instances(txn, amount, subset); + } + return instances + extra; +} + +static uint64_t inc_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + + uint64_t count; + if (dbr == MDB_NOTFOUND) + count = 1; + else + count = 1 + *(const uint64_t*)v.mv_data; + + v.mv_data = &count; + v.mv_size = sizeof(count); + dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set ring instances: " + std::string(mdb_strerror(dbr))); + + return count; +} + +static std::vector<crypto::key_image> get_key_images(MDB_txn *txn, const output_data &od) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return {}; + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + std::vector<crypto::key_image> key_images; + key_images.reserve(v.mv_size / 32); + const crypto::key_image *ki = (const crypto::key_image*)v.mv_data; + for (size_t n = 0; n < v.mv_size / 32; ++n) + key_images.push_back(*ki++); + return key_images; +} + +static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key_image &ki) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output"); + std::string data; + if (!dbr) + { + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + data = std::string((const char*)v.mv_data, v.mv_size); + } + data += std::string((const char*)&ki, sizeof(ki)); + + v.mv_data = (void*)data.data(); + v.mv_size = data.size(); + dbr = mdb_put(txn, dbi_outputs, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); +} + +static bool get_stat(MDB_txn *txn, const char *key, uint64_t &data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + int dbr = mdb_get(txn, dbi_stats, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get stat record"); + CHECK_AND_ASSERT_THROW_MES(v.mv_size == sizeof(uint64_t), "Unexpected record size"); + data = *(const uint64_t*)v.mv_data; + return true; +} + +static void set_stat(MDB_txn *txn, const char *key, uint64_t data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + v.mv_data = (void*)&data; + v.mv_size = sizeof(uint64_t); + int dbr = mdb_put(txn, dbi_stats, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set stat record"); +} + +static void inc_stat(MDB_txn *txn, const char *key) +{ + uint64_t data; + if (!get_stat(txn, key, data)) + data = 0; + ++data; + set_stat(txn, key, data); +} + +static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi) +{ + tools::create_directories_if_necessary(filename); + + int flags = MDB_RDONLY; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + int dbr = mdb_env_create(env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(*env, 1); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + MINFO("Opening monero blockchain at " << actual_filename); + dbr = mdb_env_open(*env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(*env, NULL, MDB_RDONLY, txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(*txn, "output_amounts", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, dbi); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(*txn, *dbi, compare_uint64); + + dbr = mdb_cursor_open(*txn, *dbi, cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); +} + +static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi) +{ + mdb_txn_abort(txn); + mdb_cursor_close(cur); + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct) +{ + uint64_t amount = 0; + MDB_val k = { sizeof(amount), (void*)&amount }, v; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + rct = 0; + } + else + { + if (dbr) throw std::runtime_error("Record 0 not found: " + std::string(mdb_strerror(dbr))); + mdb_size_t count = 0; + dbr = mdb_cursor_count(cur, &count); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + rct = count; + } + MDB_stat s; + dbr = mdb_stat(txn, dbi, &s); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + if (s.ms_entries < rct) throw std::runtime_error("Inconsistent records: " + std::string(mdb_strerror(dbr))); + pre_rct = s.ms_entries - rct; +} + +static crypto::hash get_genesis_block_hash(const std::string &filename) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 1); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "block_info", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi); + mdb_set_dupsort(txn, dbi, compare_uint64); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + uint64_t zero = 0; + MDB_val k = { sizeof(uint64_t), (void*)&zero}, v; + dbr = mdb_get(txn, dbi, &k, &v); + if (dbr) throw std::runtime_error("Failed to retrieve genesis block: " + std::string(mdb_strerror(dbr))); + crypto::hash genesis_block_hash = *(const crypto::hash*)(((const uint64_t*)v.mv_data) + 5); + mdb_dbi_close(env, dbi); + mdb_txn_abort(txn); + mdb_env_close(env); + tx_active = false; + return genesis_block_hash; +} + +static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const std::string &filename) +{ + std::vector<std::pair<uint64_t, uint64_t>> outputs; + uint64_t amount = std::numeric_limits<uint64_t>::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + if (!fgets(s, sizeof(s), f)) + { + MERROR("Error reading from " << filename << ": " << strerror(errno)); + break; + } + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair<uint64_t, uint64_t> output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets-- > 0) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +static bool export_spent_outputs(MDB_cursor *cur, const std::string &filename) +{ + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to open " << filename << ": " << strerror(errno)); + return false; + } + + uint64_t pending_amount = std::numeric_limits<uint64_t>::max(); + std::vector<uint64_t> pending_offsets; + MDB_val k, v; + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int dbr = mdb_cursor_get(cur, &k, &v, op); + if (dbr == MDB_NOTFOUND) + break; + op = MDB_NEXT; + if (dbr) + { + fclose(f); + MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr)); + return false; + } + const uint64_t amount = *(const uint64_t*)k.mv_data; + const uint64_t offset = *(const uint64_t*)v.mv_data; + if (!pending_offsets.empty() && (amount != pending_amount || pending_offsets.back()+1 != offset)) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + if (pending_amount != amount) + { + fprintf(f, "@%" PRIu64 "\n", amount); + pending_amount = amount; + } + pending_offsets.push_back(offset); + } + if (!pending_offsets.empty()) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + fclose(f); + return true; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -231,31 +1018,37 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor<std::string, false, true, 2> arg_blackball_db_dir = { - "blackball-db-dir", "Specify blackball database directory", + const command_line::arg_descriptor<std::string> arg_blackball_db_dir = { + "spent-output-db-dir", "Specify spent output database directory", get_default_db_path(), - {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val) { - if (testnet_stagenet[0]) - return (boost::filesystem::path(val) / "testnet").string(); - else if (testnet_stagenet[1]) - return (boost::filesystem::path(val) / "stagenet").string(); - return val; - } }; const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::string> arg_database = { "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<bool> arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; + const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output)", false}; const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor<std::string> arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; + const command_line::arg_descriptor<std::string> arg_export = {"export", "Filename to export the backball list to"}; + const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_check_subsets); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); + command_line::add_arg(desc_cmd_sett, arg_export); + command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -283,7 +1076,7 @@ int main(int argc, char* argv[]) return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-blackball.log"), true); + mlog_configure(mlog_get_default_log_path("monero-blockchain-find-spent-outputs.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else @@ -291,11 +1084,14 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Starting..."); - bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); + std::string opt_export = command_line::get_arg(vm, arg_export); + std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); + std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -304,116 +1100,99 @@ int main(int argc, char* argv[]) return 1; } - // If we wanted to use the memory pool, we would set up a fake_core. + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + if (!parse_db_sync_mode(db_sync_mode)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } - // Use Blockchain instead of lower-level BlockchainDB for two reasons: - // 1. Blockchain has the init() method for easy setup - // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() - // - // cannot match blockchain_storage setup above with just one line, - // e.g. - // Blockchain* core_storage = new Blockchain(NULL); - // because unlike blockchain_storage constructor, which takes a pointer to - // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. - LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs); if (inputs.empty()) { LOG_PRINT_L0("No inputs given"); return 1; } - std::vector<std::unique_ptr<Blockchain>> core_storage(inputs.size()); - Blockchain *blockchain = NULL; - tx_memory_pool m_mempool(*blockchain); - for (size_t n = 0; n < inputs.size(); ++n) - { - core_storage[n].reset(new Blockchain(m_mempool)); - BlockchainDB* db = new_db(db_type); - if (db == NULL) - { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); - } - LOG_PRINT_L0("database: " << db_type); + const std::string cache_dir = (output_file_path / "spent-outputs-cache").string(); + init(cache_dir); - std::string filename = inputs[n]; - while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) - filename.pop_back(); - LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + LOG_PRINT_L0("Scanning for spent outputs..."); - try - { - db->open(filename, DBF_RDONLY); - } - catch (const std::exception& e) - { - LOG_PRINT_L0("Error opening database: " << e.what()); - return 1; - } - r = core_storage[n]->init(db, net_type); + size_t done = 0; - CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); - LOG_PRINT_L0("Source blockchain storage initialized OK"); - } + const uint64_t start_blackballed_outputs = get_num_spent_outputs(); - boost::filesystem::path direc(output_file_path.string()); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - { - MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); - return 1; - } - } - else - { - if (!boost::filesystem::create_directories(direc)) - { - MERROR("Failed to create directory: " << output_file_path.string()); - return 1; - } - } + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); + + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); - LOG_PRINT_L0("Scanning for blackballable outputs..."); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); - size_t done = 0; - blackball_state_t state; - std::unordered_set<output_data> newly_spent; - const std::string state_file_path = (boost::filesystem::path(output_file_path) / "blackball-state.bin").string(); + // open first db + MDB_env *env0; + MDB_txn *txn0; + MDB_dbi dbi0; + MDB_cursor *cur0; + open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) + if (!extra_spent_outputs.empty()) { - try + MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + for (const std::pair<uint64_t, uint64_t> &output: extra_spent_outputs) { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; + if (!is_output_spent(cur, output_data(output.first, output.second))) + { + blackballs.push_back(output); + if (add_spent_output(cur, output_data(output.first, output.second))) + inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra"); + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = blackball_state_t(); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_in.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } - uint64_t start_blackballed_outputs = state.spent.size(); - - cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); - tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); for (size_t n = 0; n < inputs.size(); ++n) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); - uint64_t start_idx = 0; - auto it = state.processed_heights.find(canonical); - if (it != state.processed_heights.end()) - start_idx = it->second; + uint64_t start_idx = get_processed_txidx(canonical); + if (n > 0 && start_idx == 0) + { + start_idx = find_first_diverging_transaction(inputs[0], inputs[n]); + LOG_PRINT_L0("First diverging transaction at " << start_idx); + } LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); - for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + size_t records = 0; + const std::string filename = inputs[n]; + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + uint64_t n_txes; + for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -425,39 +1204,65 @@ int main(int argc, char* argv[]) const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); if (n == 0) for (uint64_t out: absolute) - state.outputs[output_data(txin.amount, out)].insert(txin.k_image); + add_key_image(txn, output_data(txin.amount, out), txin.k_image); + std::vector<uint64_t> relative_ring; std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - state.ring_instances[new_ring] += 1; - if (ring_size == 1) + const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); + if (n == 0 && ring_size == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); - MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[0])); - state.spent.insert(output_data(txin.amount, absolute[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a 1-ring"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, absolute[0]))) + inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } - else if (state.ring_instances[new_ring] == new_ring.size()) + else if (n == 0 && instances == new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[o])); - state.spent.insert(output_data(txin.amount, absolute[o])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, absolute[o]))) + inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } - else if (state.relative_rings.find(txin.k_image) != state.relative_rings.end()) + else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) { - MINFO("Key image " << txin.k_image << " already seen: rings " << - boost::join(state.relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + for (size_t o = 0; o < new_ring.size(); ++o) + { + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, absolute[o]))) + inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); + } + } + else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) + { + MDEBUG("Key image " << txin.k_image << " already seen: rings " << + boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); - if (state.relative_rings[txin.k_image] != txin.key_offsets) + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (relative_ring != txin.key_offsets) { - MINFO("Rings are different"); - const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[txin.k_image]); + MDEBUG("Rings are different"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring); const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); std::vector<uint64_t> common; for (uint64_t out: r0) @@ -468,18 +1273,24 @@ int main(int argc, char* argv[]) if (common.empty()) { MERROR("Rings for the same key image are disjoint"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } else if (common.size() == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); - MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, common[0])); - state.spent.insert(output_data(txin.amount, common[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in rings with a single common element"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, common[0]))) + inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else { - MINFO("The intersection has more than one element, it's still ok"); + MDEBUG("The intersection has more than one element, it's still ok"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &out: r0) if (std::find(common.begin(), common.end(), out) != common.end()) new_ring.push_back(out); @@ -487,68 +1298,172 @@ int main(int argc, char* argv[]) } } } - state.relative_rings[txin.k_image] = new_ring; + if (n == 0) + set_relative_ring(txn, txin.k_image, new_ring); + } + set_processed_txidx(txn, canonical, start_idx+1); + + ++records; + if (records >= records_per_sync) + { + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + records = 0; + } + + if (stop_requested) + { + MINFO("Stopping scan..."); + return false; } return true; }); - state.processed_heights[canonical] = start_idx; + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx); + if (stop_requested) + break; } - while (!newly_spent.empty()) + std::vector<output_data> work_spent; + + if (stop_requested) + goto skip_secondary_passes; + + if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs) + { + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + work_spent = get_spent_outputs(txn); + mdb_txn_abort(txn); + } + + while (!work_spent.empty()) { - LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); - std::unordered_set<output_data> work_spent = std::move(newly_spent); - newly_spent.clear(); + LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs"); + + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); - for (const output_data &od: work_spent) + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + std::vector<output_data> scan_spent = std::move(work_spent); + work_spent.clear(); + for (const output_data &od: scan_spent) { - for (const crypto::key_image &ki: state.outputs[od]) + std::vector<crypto::key_image> key_images = get_key_images(txn, od); + for (const crypto::key_image &ki: key_images) { - std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[ki]); + std::vector<uint64_t> relative_ring; + CHECK_AND_ASSERT_THROW_MES(get_relative_ring(txn, ki, relative_ring), "Relative ring not found"); + std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_ring); size_t known = 0; uint64_t last_unknown = 0; for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (state.spent.find(new_od) != state.spent.end()) + if (is_output_spent(cur, new_od)) ++known; else last_unknown = out; } if (known == absolute.size() - 1) { - const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); - MINFO("Blackballing output " << pkey << ", due to being used in a " << - absolute.size() << "-ring where all other outputs are known to be spent"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(od.amount, last_unknown)); - state.spent.insert(output_data(od.amount, last_unknown)); + const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(od.amount, last_unknown))) + inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); + work_spent.push_back(output_data(od.amount, last_unknown)); } } - } - } - LOG_PRINT_L0("Saving state data to " << state_file_path); - std::ofstream state_data_out; - state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); - if (!state_data_out.fail()) - { - try - { - boost::archive::portable_binary_oarchive a(state_data_out); - a << state; + if (stop_requested) + { + MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully."); + return 0; + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to save state data to " << state_file_path); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_out.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + } + +skip_secondary_passes: + uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs; + LOG_PRINT_L0(std::to_string(diff) << " new outputs marked as spent, " << get_num_spent_outputs() << " total outputs marked as spent"); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + uint64_t pre_rct = 0, rct = 0; + get_num_outputs(txn0, cur0, dbi0, pre_rct, rct); + MINFO("Total pre-rct outputs: " << pre_rct); + MINFO("Total rct outputs: " << rct); + static const struct { const char *key; uint64_t base; } stat_keys[] = { + { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, + { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, + { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, + { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, + { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, + { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, + }; + for (const auto &key: stat_keys) + { + uint64_t data; + if (!get_stat(txn, key.key, data)) + data = 0; + float percent = key.base ? 100.0f * data / key.base : 0.0f; + MINFO(key.key << ": " << data << " (" << percent << "%)"); + } + mdb_txn_abort(txn); + + if (!opt_export.empty()) + { + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + export_spent_outputs(cur, opt_export); + mdb_cursor_close(cur); + mdb_txn_abort(txn); } - uint64_t diff = state.spent.size() - start_blackballed_outputs; - LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << state.spent.size() << " total outputs blackballed"); - LOG_PRINT_L0("Blockchain blackball data exported OK"); + LOG_PRINT_L0("Blockchain spent output data exported OK"); + close_db(env0, txn0, cur0, dbi0); + close(); return 0; - CATCH_ENTRY("Export error", 1); + CATCH_ENTRY("Error", 1); } diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp new file mode 100644 index 000000000..8060b0de4 --- /dev/null +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -0,0 +1,350 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get min depth for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get min depth for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + + if (!opt_txid_string.empty() && opt_height) + { + std::cerr << "txid and height cannot be given at the same time" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + if (opt_include_coinbase) + start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + std::vector<uint64_t> depths; + for (const crypto::hash &start_txid: start_txids) + { + uint64_t depth = 0; + bool coinbase = false; + + LOG_PRINT_L0("Checking depth for txid " << start_txid); + std::vector<crypto::hash> txids(1, start_txid); + while (!coinbase) + { + LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth); + std::vector<crypto::hash> new_txids; + for (const crypto::hash &txid: txids) + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_gen)) + { + MDEBUG(txid << " is a coinbase transaction"); + coinbase = true; + goto done; + } + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + if (!coinbase) + { + std::swap(txids, new_txids); + ++depth; + } + } +done: + LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); + depths.push_back(depth); + } + + uint64_t cumulative_depth = 0; + for (uint64_t depth: depths) + cumulative_depth += depth; + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size()); + LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << epee::misc_utils::median(depths)); + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index caa549c13..7f92ecd87 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -33,9 +33,11 @@ #include <boost/filesystem.hpp> #include <boost/algorithm/string.hpp> +#include <unistd.h> #include "misc_log_ex.h" #include "bootstrap_file.h" #include "bootstrap_serialization.h" +#include "blocks/blocks.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/json_utils.h" // dump_json() @@ -164,7 +166,7 @@ int pop_blocks(cryptonote::core& core, int num_blocks) return num_blocks; } -int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks, bool force) +int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &blocks, bool force) { if (blocks.empty()) return 0; @@ -176,7 +178,7 @@ int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks, if (!force && new_height % HASH_OF_HASHES_STEP) return 0; - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; for (const auto &b: blocks) { cryptonote::block block; @@ -312,7 +314,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path MINFO("Reading blockchain from bootstrap file..."); std::cout << ENDL; - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; // Skip to start_height before we start adding. { @@ -437,7 +439,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { cryptonote::blobdata block; cryptonote::block_to_blob(bp.block, block); - std::list<cryptonote::blobdata> txs; + std::vector<cryptonote::blobdata> txs; for (const auto &tx: bp.txs) { txs.push_back(cryptonote::blobdata()); @@ -473,17 +475,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path txs.push_back(tx); } - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; - block_size = bp.block_size; + block_weight = bp.block_weight; cumulative_difficulty = bp.cumulative_difficulty; coins_generated = bp.coins_generated; try { - core.get_blockchain_storage().get_db().add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { @@ -593,8 +595,8 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<std::string> arg_database = { "database", available_dbs.c_str(), default_db_type }; - const command_line::arg_descriptor<bool> arg_verify = {"guard-against-pwnage", - "Verify blocks and transactions during import (only disable if you exported the file yourself)", true}; + const command_line::arg_descriptor<bool> arg_noverify = {"dangerous-unverified-import", + "Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)", false}; const command_line::arg_descriptor<bool> arg_batch = {"batch", "Batch transactions for faster import", true}; const command_line::arg_descriptor<bool> arg_resume = {"resume", @@ -614,7 +616,7 @@ int main(int argc, char* argv[]) // call add_options() directly for these arguments since // command_line helpers support only boolean switch, not boolean argument desc_cmd_sett.add_options() - (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_noverify.name, make_semantic(arg_noverify), arg_noverify.description) (arg_batch.name, make_semantic(arg_batch), arg_batch.description) (arg_resume.name, make_semantic(arg_resume), arg_resume.description) ; @@ -633,7 +635,7 @@ int main(int argc, char* argv[]) if (! r) return 1; - opt_verify = command_line::get_arg(vm, arg_verify); + opt_verify = !command_line::get_arg(vm, arg_noverify); opt_batch = command_line::get_arg(vm, arg_batch); opt_resume = command_line::get_arg(vm, arg_resume); block_stop = command_line::get_arg(vm, arg_block_stop); @@ -738,6 +740,18 @@ int main(int argc, char* argv[]) MINFO("bootstrap file path: " << import_file_path); MINFO("database path: " << m_config_folder); + if (!opt_verify) + { + MCLOG_RED(el::Level::Warning, "global", "\n" + "Import is set to proceed WITHOUT VERIFICATION.\n" + "This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n" + "you could end up with a compromised database. It is recommended to NOT use " << arg_noverify.name << ".\n" + "*****************************************************************************************\n" + "You have 90 seconds to press ^C or terminate this program before unverified import starts\n" + "*****************************************************************************************"); + sleep(90); + } + cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cryptonote::core core(&pr); @@ -745,7 +759,12 @@ int main(int argc, char* argv[]) { core.disable_dns_checkpoints(true); - if (!core.init(vm, NULL)) +#if defined(PER_BLOCK_CHECKPOINT) + const GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData; +#else + const GetCheckpointsCallback& get_checkpoints = nullptr; +#endif + if (!core.init(vm, nullptr, nullptr, get_checkpoints)) { std::cerr << "Failed to initialize core" << ENDL; return 1; diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp new file mode 100644 index 000000000..716b33cae --- /dev/null +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -0,0 +1,337 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + uint64_t block_start = 0; + uint64_t block_stop = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "start at block number", block_start}; + const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop}; + const command_line::arg_descriptor<bool> arg_inputs = {"with-inputs", "with input stats", false}; + const command_line::arg_descriptor<bool> arg_outputs = {"with-outputs", "with output stats", false}; + const command_line::arg_descriptor<bool> arg_ringsize = {"with-ringsize", "with ringsize stats", false}; + const command_line::arg_descriptor<bool> arg_hours = {"with-hours", "with txns per hour", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_block_start); + command_line::add_arg(desc_cmd_sett, arg_block_stop); + command_line::add_arg(desc_cmd_sett, arg_inputs); + command_line::add_arg(desc_cmd_sett, arg_outputs); + command_line::add_arg(desc_cmd_sett, arg_ringsize); + command_line::add_arg(desc_cmd_sett, arg_hours); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-stats.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + block_start = command_line::get_arg(vm, arg_block_start); + block_stop = command_line::get_arg(vm, arg_block_stop); + bool do_inputs = command_line::get_arg(vm, arg_inputs); + bool do_outputs = command_line::get_arg(vm, arg_outputs); + bool do_ringsize = command_line::get_arg(vm, arg_ringsize); + bool do_hours = command_line::get_arg(vm, arg_hours); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + const uint64_t db_height = db->height(); + if (!block_stop) + block_stop = db_height; + MINFO("Starting from height " << block_start << ", stopping at height " << block_stop); + +/* + * The default output can be plotted with GnuPlot using these commands: +set key autotitle columnhead +set title "Monero Blockchain Growth" +set timefmt "%Y-%m-%d" +set xdata time +set xrange ["2014-04-17":*] +set format x "%Y-%m-%d" +set yrange [0:*] +set y2range [0:*] +set ylabel "Txs/Day" +set y2label "Bytes" +set y2tics nomirror +plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' using (timecolumn(1,"%Y-%m-%d")):7 axes x1y2 with lines + */ + + // spit out a comment that GnuPlot can use as an index + std::cout << ENDL << "# DATA" << ENDL; + std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes"; + if (do_inputs) + std::cout << "\tInMin\tInMax\tInAvg"; + if (do_outputs) + std::cout << "\tOutMin\tOutMax\tOutAvg"; + if (do_ringsize) + std::cout << "\tRingMin\tRingMax\tRingAvg"; + if (do_hours) { + char buf[8]; + unsigned int i; + for (i=0; i<24; i++) { + sprintf(buf, "\t%02d:00", i); + std::cout << buf; + } + } + std::cout << ENDL; + + struct tm prevtm = {0}, currtm; + uint64_t prevsz = 0, currsz = 0; + uint64_t prevtxs = 0, currtxs = 0; + uint64_t currblks = 0; + uint64_t totins = 0, totouts = 0, totrings = 0; + uint32_t minins = 10, maxins = 0; + uint32_t minouts = 10, maxouts = 0; + uint32_t minrings = 50, maxrings = 0; + uint32_t io, tottxs = 0; + uint32_t txhr[24] = {0}; + unsigned int i; + + for (uint64_t h = block_start; h < block_stop; ++h) + { + cryptonote::blobdata bd = db->get_block_blob_from_height(h); + cryptonote::block blk; + if (!cryptonote::parse_and_validate_block_from_blob(bd, blk)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + time_t tt = blk.timestamp; + char timebuf[64]; + gmtime_r(&tt, &currtm); + if (!prevtm.tm_year) + prevtm = currtm; + // catch change of day + if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27)) + { + // check for timestamp fudging around month ends + if (prevtm.tm_mday == 1 && currtm.tm_mday > 27) + goto skip; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); + prevtm = currtm; + std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; + prevsz += currsz; + currsz = 0; + currblks = 0; + prevtxs += currtxs; + currtxs = 0; + if (!tottxs) + tottxs = 1; + if (do_inputs) { + std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs; + minins = 10; maxins = 0; totins = 0; + } + if (do_outputs) { + std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs; + minouts = 10; maxouts = 0; totouts = 0; + } + if (do_ringsize) { + std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs; + minrings = 50; maxrings = 0; totrings = 0; + } + tottxs = 0; + if (do_hours) { + for (i=0; i<24; i++) { + std::cout << "\t" << txhr[i]; + txhr[i] = 0; + } + } + std::cout << ENDL; + } +skip: + currsz += bd.size(); + for (const auto& tx_id : blk.tx_hashes) + { + if (tx_id == crypto::null_hash) + { + throw std::runtime_error("Aborting: tx == null_hash"); + } + if (!db->get_tx_blob(tx_id, bd)) + { + throw std::runtime_error("Aborting: tx not found"); + } + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad txn from db"); + return 1; + } + currsz += bd.size(); + currtxs++; + if (do_hours) + txhr[currtm.tm_hour]++; + if (do_inputs) { + io = tx.vin.size(); + if (io < minins) + minins = io; + else if (io > maxins) + maxins = io; + totins += io; + } + if (do_ringsize) { + const cryptonote::txin_to_key& tx_in_to_key + = boost::get<cryptonote::txin_to_key>(tx.vin[0]); + io = tx_in_to_key.key_offsets.size(); + if (io < minrings) + minrings = io; + else if (io > maxrings) + maxrings = io; + totrings += io; + } + if (do_outputs) { + io = tx.vout.size(); + if (io < minouts) + minouts = io; + else if (io > maxouts) + maxouts = io; + totouts += io; + } + tottxs++; + } + currblks++; + + if (stop_requested) + break; + } + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Stats reporting error", 1); +} diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index aa85e5e53..beaad2abc 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -236,11 +236,11 @@ void BootstrapFile::write_block(block& block) bool include_extra_block_data = true; if (include_extra_block_data) { - size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); + size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height); difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); - bp.block_size = block_size; + bp.block_weight = block_weight; bp.cumulative_difficulty = cumulative_difficulty; bp.coins_generated = coins_generated; } @@ -400,18 +400,18 @@ uint64_t BootstrapFile::count_bytes(std::ifstream& import_file, uint64_t blocks, { std::cout << refresh_string; MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE - << " height: " << h-1); + << " height: " << h-1 << ", offset " << bytes_read); throw std::runtime_error("Aborting: chunk size exceeds buffer size"); } if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD) { std::cout << refresh_string; MDEBUG("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD << " << height: " - << h-1); + << h-1 << ", offset " << bytes_read); } else if (chunk_size <= 0) { std::cout << refresh_string; - MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1); + MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1 << ", offset " << bytes_read); throw std::runtime_error("Aborting"); } // skip to next expected block size value diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 8755d3fe3..278a7b40f 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -70,14 +70,14 @@ namespace cryptonote { cryptonote::block block; std::vector<transaction> txs; - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; BEGIN_SERIALIZE() FIELD(block) FIELD(txs) - VARINT_FIELD(block_size) + VARINT_FIELD(block_weight) VARINT_FIELD(cumulative_difficulty) VARINT_FIELD(coins_generated) END_SERIALIZE() diff --git a/src/blocks/CMakeLists.txt b/src/blocks/CMakeLists.txt index cf3f0b354..30d85adbf 100644 --- a/src/blocks/CMakeLists.txt +++ b/src/blocks/CMakeLists.txt @@ -26,14 +26,23 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -if(APPLE) - add_library(blocks STATIC blockexports.c) - set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C) -else() - add_custom_command(OUTPUT blocks.o MAIN_DEPENDENCY blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat) - add_custom_command(OUTPUT testnet_blocks.o MAIN_DEPENDENCY testnet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/testnet_blocks.o testnet_blocks.dat) - add_custom_command(OUTPUT stagenet_blocks.o MAIN_DEPENDENCY stagenet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/stagenet_blocks.o stagenet_blocks.dat) - add_library(blocks STATIC blocks.o testnet_blocks.o stagenet_blocks.o blockexports.c) - set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C) -endif() +set(GENERATED_SOURCES "") +foreach(BLOB_NAME checkpoints testnet_blocks stagenet_blocks) + set(OUTPUT_C_SOURCE "generated_${BLOB_NAME}.c") + list(APPEND GENERATED_SOURCES ${OUTPUT_C_SOURCE}) + set(INPUT_DAT_FILE "${BLOB_NAME}.dat") + add_custom_command( + OUTPUT ${OUTPUT_C_SOURCE} + MAIN_DEPENDENCY ${INPUT_DAT_FILE} + COMMAND + cd ${CMAKE_CURRENT_BINARY_DIR} && + echo "'#include\t<stddef.h>'" > ${OUTPUT_C_SOURCE} && + echo "'const\tunsigned\tchar\t${BLOB_NAME}[]={'" >> ${OUTPUT_C_SOURCE} && + od -v -An -tu1 ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_DAT_FILE} | sed -e "'s/[0-9]\\{1,\\}/&,/g'" -e "'$$s/.$$//'" >> ${OUTPUT_C_SOURCE} && + echo "'};'" >> ${OUTPUT_C_SOURCE} && + echo "'const\tsize_t\t${BLOB_NAME}_len\t=\tsizeof(${BLOB_NAME});'" >> ${OUTPUT_C_SOURCE} + ) +endforeach() + +add_library(blocks STATIC blocks.cpp ${GENERATED_SOURCES}) diff --git a/src/blocks/blockexports.c b/src/blocks/blockexports.c deleted file mode 100644 index 0154b0413..000000000 --- a/src/blocks/blockexports.c +++ /dev/null @@ -1,87 +0,0 @@ -#include <stddef.h> - -#if defined(__APPLE__) -#include <mach-o/getsect.h> -#ifdef BUILD_SHARED_LIBS -#if !defined(__LP64__) -const struct mach_header _mh_execute_header; -#else -const struct mach_header_64 _mh_execute_header; -#endif -#else -#if !defined(__LP64__) -extern const struct mach_header _mh_execute_header; -#else -extern const struct mach_header_64 _mh_execute_header; -#endif -#endif - -const unsigned char *get_blocks_dat_start(int testnet, int stagenet) -{ - size_t size; - if (testnet) - return getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size); - else if (stagenet) - return getsectiondata(&_mh_execute_header, "__DATA", "__stagenet_blocks_dat", &size); - else - return getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size); -} - -size_t get_blocks_dat_size(int testnet, int stagenet) -{ - size_t size; - if (testnet) - getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size); - else if (stagenet) - getsectiondata(&_mh_execute_header, "__DATA", "__stagenet_blocks_dat", &size); - else - getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size); - return size; -} - -#else - -#if defined(_WIN32) && !defined(_WIN64) -#define _binary_blocks_start binary_blocks_dat_start -#define _binary_blocks_end binary_blocks_dat_end -#define _binary_testnet_blocks_start binary_testnet_blocks_dat_start -#define _binary_testnet_blocks_end binary_testnet_blocks_dat_end -#define _binary_stagenet_blocks_start binary_stagenet_blocks_dat_start -#define _binary_stagenet_blocks_end binary_stagenet_blocks_dat_end -#else -#define _binary_blocks_start _binary_blocks_dat_start -#define _binary_blocks_end _binary_blocks_dat_end -#define _binary_testnet_blocks_start _binary_testnet_blocks_dat_start -#define _binary_testnet_blocks_end _binary_testnet_blocks_dat_end -#define _binary_stagenet_blocks_start _binary_stagenet_blocks_dat_start -#define _binary_stagenet_blocks_end _binary_stagenet_blocks_dat_end -#endif - -extern const unsigned char _binary_blocks_start[]; -extern const unsigned char _binary_blocks_end[]; -extern const unsigned char _binary_testnet_blocks_start[]; -extern const unsigned char _binary_testnet_blocks_end[]; -extern const unsigned char _binary_stagenet_blocks_start[]; -extern const unsigned char _binary_stagenet_blocks_end[]; - -const unsigned char *get_blocks_dat_start(int testnet, int stagenet) -{ - if (testnet) - return _binary_testnet_blocks_start; - else if (stagenet) - return _binary_stagenet_blocks_start; - else - return _binary_blocks_start; -} - -size_t get_blocks_dat_size(int testnet, int stagenet) -{ - if (testnet) - return (size_t) (_binary_testnet_blocks_end - _binary_testnet_blocks_start); - else if (stagenet) - return (size_t) (_binary_stagenet_blocks_end - _binary_stagenet_blocks_start); - else - return (size_t) (_binary_blocks_end - _binary_blocks_start); -} - -#endif diff --git a/src/blocks/blocks.cpp b/src/blocks/blocks.cpp new file mode 100644 index 000000000..0661f8448 --- /dev/null +++ b/src/blocks/blocks.cpp @@ -0,0 +1,31 @@ +#include "blocks.h" + +#include <unordered_map> + +extern const unsigned char checkpoints[]; +extern const size_t checkpoints_len; +extern const unsigned char stagenet_blocks[]; +extern const size_t stagenet_blocks_len; +extern const unsigned char testnet_blocks[]; +extern const size_t testnet_blocks_len; + +namespace blocks +{ + + const std::unordered_map<cryptonote::network_type, const epee::span<const unsigned char>, std::hash<size_t>> CheckpointsByNetwork = { + {cryptonote::network_type::MAINNET, {checkpoints, checkpoints_len}}, + {cryptonote::network_type::STAGENET, {stagenet_blocks, stagenet_blocks_len}}, + {cryptonote::network_type::TESTNET, {testnet_blocks, testnet_blocks_len}} + }; + + const epee::span<const unsigned char> GetCheckpointsData(cryptonote::network_type network) + { + const auto it = CheckpointsByNetwork.find(network); + if (it != CheckpointsByNetwork.end()) + { + return it->second; + } + return nullptr; + } + +} diff --git a/src/blocks/blocks.dat b/src/blocks/blocks.dat deleted file mode 100644 index e69de29bb..000000000 --- a/src/blocks/blocks.dat +++ /dev/null diff --git a/src/blocks/blocks.h b/src/blocks/blocks.h index ec683f47e..14e391319 100644 --- a/src/blocks/blocks.h +++ b/src/blocks/blocks.h @@ -1,16 +1,12 @@ #ifndef SRC_BLOCKS_BLOCKS_H_ #define SRC_BLOCKS_BLOCKS_H_ -#ifdef __cplusplus -extern "C" { -#endif +#include "cryptonote_config.h" +#include "span.h" -const unsigned char *get_blocks_dat_start(int testnet, int stagenet); -size_t get_blocks_dat_size(int testnet, int stagenet); - -#ifdef __cplusplus +namespace blocks +{ + const epee::span<const unsigned char> GetCheckpointsData(cryptonote::network_type network); } -#endif - #endif /* SRC_BLOCKS_BLOCKS_H_ */ diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat Binary files differindex 085558504..03e2cd2a8 100644 --- a/src/blocks/checkpoints.dat +++ b/src/blocks/checkpoints.dat diff --git a/src/checkpoints/CMakeLists.txt b/src/checkpoints/CMakeLists.txt index 02bb2891a..715006522 100644 --- a/src/checkpoints/CMakeLists.txt +++ b/src/checkpoints/CMakeLists.txt @@ -27,9 +27,13 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. if(APPLE) - find_library(IOKIT_LIBRARY IOKit) - mark_as_advanced(IOKIT_LIBRARY) - list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + if(DEPENDS) + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework ApplicationServices -framework AppKit -framework IOKit") + else() + find_library(IOKIT_LIBRARY IOKit) + mark_as_advanced(IOKIT_LIBRARY) + list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + endif() endif() set(checkpoints_sources diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index ef1ee171d..6251fcc91 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -165,6 +165,7 @@ namespace cryptonote { ADD_CHECKPOINT(0, "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b"); ADD_CHECKPOINT(1000000, "46b690b710a07ea051bc4a6b6842ac37be691089c0f7758cfeec4d5fc0b4a258"); + ADD_CHECKPOINT(1058600, "12904f6b4d9e60fd875674e07147d2c83d6716253f046af7b894c3e81da7e1bd"); return true; } if (nettype == STAGENET) @@ -208,7 +209,7 @@ namespace cryptonote ADD_CHECKPOINT(1450000, "ac94e8860093bc7c83e4e91215cba1d663421ecf4067a0ae609c3a8b52bcfac2"); ADD_CHECKPOINT(1530000, "01759bce497ec38e63c78b1038892169203bb78f87e488172f6b854fcd63ba7e"); ADD_CHECKPOINT(1579000, "7d0d7a2346373afd41ed1e744a939fc5d474a7dbaa257be5c6fff4009e789241"); - + ADD_CHECKPOINT(1668900, "ac2dcaf3d2f58ffcf8391639f0f1ebafcb8eac43c49479c7c37f611868d07568"); return true; } diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 808ef7630..aed9bfee7 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -33,12 +33,17 @@ set(common_sources command_line.cpp dns_utils.cpp download.cpp + error.cpp + expect.cpp util.cpp i18n.cpp + notify.cpp password.cpp perf_timer.cpp + spawn.cpp threadpool.cpp - updates.cpp) + updates.cpp + aligned.c) if (STACK_TRACE) list(APPEND common_sources stack_trace.cpp) @@ -54,8 +59,11 @@ set(common_private_headers common_fwd.h dns_utils.h download.h + error.h + expect.h http_connection.h int-util.h + notify.h pod-class.h rpc_client.h scoped_message_writer.h @@ -65,9 +73,11 @@ set(common_private_headers i18n.h password.h perf_timer.h + spawn.h stack_trace.h threadpool.h - updates.h) + updates.h + aligned.h) monero_private_headers(common ${common_private_headers}) @@ -78,7 +88,6 @@ monero_add_library(common DEPENDS generate_translations_header) target_link_libraries(common PUBLIC - epee cncrypto ${UNBOUND_LIBRARY} ${LIBUNWIND_LIBRARIES} diff --git a/src/common/aligned.c b/src/common/aligned.c new file mode 100644 index 000000000..763dfd0e7 --- /dev/null +++ b/src/common/aligned.c @@ -0,0 +1,139 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <stdint.h> +#include <string.h> +#include "aligned.h" + +static inline int is_power_of_2(size_t n) { return n && (n & (n-1)) == 0; } + +#define MAGIC 0xaa0817161500ff81 +#define MAGIC_FREED 0xaa0817161500ff82 + +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +typedef struct +{ + uint64_t magic; + void *raw; + size_t bytes; + size_t align; +} control; + +void *aligned_malloc(size_t bytes, size_t align) +{ + void *raw, *ptr; + control *ctrl; + + if (!is_power_of_2(align)) + return NULL; + if (bytes > (size_t)-1 - align) + return NULL; + if (bytes + align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + align); + if (!raw) + return NULL; + ptr = (void*)(((uintptr_t)raw + align + sizeof(control) - 1) & ~(align-1)); + ctrl = ((control*)ptr) - 1; + ctrl->magic = MAGIC; + ctrl->raw = raw; + ctrl->bytes = bytes; + ctrl->align = align; + return ptr; +} + +void *aligned_realloc(void *ptr, size_t bytes, size_t align) +{ + void *raw, *ptr2; + control *ctrl, *ctrl2; + + if (!ptr) + return aligned_malloc(bytes, align); + if (!bytes) + { + aligned_free(ptr); + return NULL; + } + if (!is_power_of_2(align)) + return NULL; + + ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + if (ctrl->align != align) + return NULL; + if (ctrl->bytes >= bytes) + return ptr; + + if (ctrl->bytes > (size_t)-1 - ctrl->align) + return NULL; + if (ctrl->bytes + ctrl->align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + ctrl->align); + if (!raw) + return NULL; + ptr2 = (void*)(((uintptr_t)raw + ctrl->align + sizeof(control) - 1) & ~(ctrl->align-1)); + memcpy(ptr2, ptr, ctrl->bytes); + ctrl2 = ((control*)ptr2) - 1; + ctrl2->magic = MAGIC; + ctrl2->raw = raw; + ctrl2->bytes = bytes; + ctrl2->align = ctrl->align; + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); + return ptr2; +} + +void aligned_free(void *ptr) +{ + if (!ptr) + return; + control *ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); +} diff --git a/src/common/aligned.h b/src/common/aligned.h new file mode 100644 index 000000000..fed3ccb36 --- /dev/null +++ b/src/common/aligned.h @@ -0,0 +1,41 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +void *aligned_malloc(size_t bytes, size_t align); +void *aligned_realloc(void *ptr, size_t bytes, size_t align); +void aligned_free(void *ptr); + +#ifdef __cplusplus +} +#endif diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 33f60bc3c..f2b270981 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -46,10 +46,11 @@ namespace bf = boost::filesystem; static const char *DEFAULT_DNS_PUBLIC_ADDR[] = { "194.150.168.168", // CCC (Germany) - "81.3.27.54", // Lightning Wire Labs (Germany) - "31.3.135.232", // OpenNIC (Switzerland) "80.67.169.40", // FDN (France) - "209.58.179.186", // Cyberghost (Singapore) + "89.233.43.71", // http://censurfridns.dk (Denmark) + "109.69.8.51", // punCAT (Spain) + "77.109.148.137", // Xiala.net (Switzerland) + "193.58.251.251", // SkyDNS (Russia) }; static boost::mutex instance_lock; @@ -97,11 +98,16 @@ get_builtin_cert(void) */ /** return the built in root DS trust anchor */ -static const char* +static const char* const* get_builtin_ds(void) { - return -". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n"; + static const char * const ds[] = + { + ". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n", + ". IN DS 20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", + NULL + }; + return ds; } /************************************************************ @@ -240,7 +246,12 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - ub_ctx_add_ta(m_data->m_ub_context, string_copy(::get_builtin_ds())); + const char * const *ds = ::get_builtin_ds(); + while (*ds) + { + MINFO("adding trust anchor: " << *ds); + ub_ctx_add_ta(m_data->m_ub_context, string_copy(*ds++)); + } } DNSResolver::~DNSResolver() diff --git a/src/common/error.cpp b/src/common/error.cpp new file mode 100644 index 000000000..e091e4478 --- /dev/null +++ b/src/common/error.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "error.h" + +#include <string> + +namespace +{ + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "common_category()"; + } + + virtual std::string message(int value) const override final + { + switch (common_error(value)) + { + case common_error::kInvalidArgument: + return make_error_code(std::errc::invalid_argument).message(); + case common_error::kInvalidErrorCode: + return "expect<T> was given an error value of zero"; + default: + break; + } + return "Unknown basic_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + // maps specific errors to generic `std::errc` cases. + switch (common_error(value)) + { + case common_error::kInvalidArgument: + case common_error::kInvalidErrorCode: + return std::errc::invalid_argument; + default: + break; + } + return std::error_condition{value, *this}; + } + }; +} + +std::error_category const& common_category() noexcept +{ + static const category instance{}; + return instance; +} + diff --git a/src/common/error.h b/src/common/error.h new file mode 100644 index 000000000..6fef3eb4b --- /dev/null +++ b/src/common/error.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <system_error> +#include <type_traits> + +enum class common_error : int +{ + // 0 is reserved for no error, as per expect<T> + kInvalidArgument = 1, //!< A function argument is invalid + kInvalidErrorCode //!< Default `std::error_code` given to `expect<T>` +}; + +std::error_category const& common_category() noexcept; + +inline std::error_code make_error_code(::common_error value) noexcept +{ + return std::error_code{int(value), common_category()}; +} + +namespace std +{ + template<> + struct is_error_code_enum<::common_error> + : true_type + {}; +} diff --git a/src/common/expect.cpp b/src/common/expect.cpp new file mode 100644 index 000000000..c86e23e95 --- /dev/null +++ b/src/common/expect.cpp @@ -0,0 +1,70 @@ +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "expect.h" + +#include <easylogging++.h> +#include <string> + +namespace detail +{ + namespace + { + std::string generate_error(const char* msg, const char* file, unsigned line) + { + std::string error_msg{}; + if (msg) + { + error_msg.append(msg); + if (file) + error_msg.append(" ("); + } + if (file) + { + error_msg.append("thrown at "); + + // remove path, get just filename + extension + char buff[256] = {0}; + el::base::utils::File::buildBaseFilename(file, buff, sizeof(buff) - 1); + error_msg.append(buff); + + error_msg.push_back(':'); + error_msg.append(std::to_string(line)); + } + if (msg && file) + error_msg.push_back(')'); + return error_msg; + } + } + + void expect::throw_(std::error_code ec, const char* msg, const char* file, unsigned line) + { + if (msg || file) + throw std::system_error{ec, generate_error(msg, file, line)}; + throw std::system_error{ec}; + } +} // detail diff --git a/src/common/expect.h b/src/common/expect.h new file mode 100644 index 000000000..72e4060a7 --- /dev/null +++ b/src/common/expect.h @@ -0,0 +1,449 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cassert> +#include <system_error> +#include <type_traits> +#include <utility> + +#include "common/error.h" + +//! If precondition fails, return `::error::kInvalidArgument` in current scope. +#define MONERO_PRECOND(...) \ + do \ + { \ + if (!( __VA_ARGS__ )) \ + return {::common_error::kInvalidArgument}; \ + } while (0) + +//! Check `expect<void>` and return errors in current scope. +#define MONERO_CHECK(...) \ + do \ + { \ + const ::expect<void> result = __VA_ARGS__ ; \ + if (!result) \ + return result.error(); \ + } while (0) + +/*! Get `T` from `expect<T>` by `std::move` as-if by function call. + `expect<void>` returns nothing. + + \throw std::system_error with `expect<T>::error()`, filename and line + number when `expect<T>::has_error() == true`.*/ +#define MONERO_UNWRAP(...) \ + ::detail::expect::unwrap( __VA_ARGS__ , nullptr, __FILE__ , __LINE__ ) + +/* \throw std::system_error with `code` and `msg` as part of the details. The +filename and line number will automatically be injected into the explanation +string. `code` can be any enum convertible to `std::error_code`. */ +#define MONERO_THROW(code, msg) \ + ::detail::expect::throw_( code , msg , __FILE__ , __LINE__ ) + + +template<typename> class expect; + +namespace detail +{ + // Shortens the characters in the places that `enable_if` is used below. + template<bool C> + using enable_if = typename std::enable_if<C>::type; + + struct expect + { + //! \throw std::system_error with `ec`, optional `msg` and/or optional `file` + `line`. + static void throw_(std::error_code ec, const char* msg, const char* file, unsigned line); + + //! If `result.has_error()` call `throw_`. Otherwise, \return `*result` by move. + template<typename T> + static T unwrap(::expect<T>&& result, const char* error_msg, const char* file, unsigned line) + { + if (!result) + throw_(result.error(), error_msg, file, line); + return std::move(*result); + } + + //! If `result.has_error()` call `throw_`. + static void unwrap(::expect<void>&& result, const char* error_msg, const char* file, unsigned line); + }; +} + +/*! + `expect<T>` is a value or error implementation, similar to Rust std::result + or various C++ proposals (boost::expected, boost::outcome). This + implementation currently has a strict error type, `std::error_code`, and a + templated value type `T`. `expect<T>` is implicitly convertible from `T` + or `std::error_code`, and one `expect<T>` object type is implicitly + convertible to another `expect<U>` object iff the destination value type + can be implicitly constructed from the source value type (i.e. + `struct U { ... U(T src) { ...} ... };`). + + `operator==` and `operator!=` are the only comparison operators provided; + comparison between different value types is allowed provided the two values + types have a `operator==` defined between them (i.e. + `assert(expect<int>{100} == expect<short>{100});`). Comparisons can also be + done against `std::error_code` objects or error code enums directly (i.e. + `assert(expect<int>{make_error_code(common_error::kInvalidArgument)} == error::kInvalidArgument)`). + Comparison of default constructed `std::error_code` will always fail. + "Generic" comparisons can be done with `std::error_condition` via the `matches` + method only (i.e. + `assert(expect<int>{make_error_code{common_error::kInvalidErrorCode}.matches(std::errc::invalid_argument))`), + `operator==` and `operator!=` will not work with `std::errc` or + `std::error_condition`. A comparison with `matches` is more expensive + because an equivalency between error categories is computed, but is + recommended when an error can be one of several categories (this is going + to be the case in nearly every situation when calling a function from + another C++ struct/class). + + `expect<void>` is a special case with no stored value. It is used by + functions that can fail, but otherwise would return `void`. It is useful + for consistency; all macros, standalone functions, and comparison operators + work with `expect<void>`. + + \note See `src/common/error.h` for creating a custom error enum. + */ +template<typename T> +class expect +{ + static_assert(std::is_nothrow_destructible<T>(), "T must have a nothrow destructor"); + + template<typename U> + static constexpr bool is_convertible() noexcept + { + return std::is_constructible<T, U>() && + std::is_convertible<U, T>(); + } + + // MEMBERS + std::error_code code_; + typename std::aligned_storage<sizeof(T), alignof(T)>::type storage_; + // MEMBERS + + T& get() noexcept + { + assert(has_value()); + return *reinterpret_cast<T*>(std::addressof(storage_)); + } + + T const& get() const noexcept + { + assert(has_value()); + return *reinterpret_cast<T const*>(std::addressof(storage_)); + } + + template<typename U> + void store(U&& value) noexcept(std::is_nothrow_constructible<T, U>()) + { + new (std::addressof(storage_)) T{std::forward<U>(value)}; + code_ = std::error_code{}; + } + + void maybe_throw() const + { + if (has_error()) + ::detail::expect::throw_(error(), nullptr, nullptr, 0); + } + +public: + using value_type = T; + using error_type = std::error_code; + + expect() = delete; + + /*! Store an error, `code`, in the `expect` object. If `code` creates a + `std::error_code` object whose `.value() == 0`, then `error()` will be set + to `::common_error::kInvalidErrorCode`. */ + expect(std::error_code const& code) noexcept + : code_(code), storage_() + { + if (!has_error()) + code_ = ::common_error::kInvalidErrorCode; + } + + //! Store a value, `val`, in the `expect` object. + expect(T val) noexcept(std::is_nothrow_move_constructible<T>()) + : code_(), storage_() + { + store(std::move(val)); + } + + expect(expect const& src) noexcept(std::is_nothrow_copy_constructible<T>()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(src.get()); + } + + //! Copy conversion from `U` to `T`. + template<typename U, typename = detail::enable_if<is_convertible<U const&>()>> + expect(expect<U> const& src) noexcept(std::is_nothrow_constructible<T, U const&>()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(*src); + } + + expect(expect&& src) noexcept(std::is_nothrow_move_constructible<T>()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(std::move(src.get())); + } + + //! Move conversion from `U` to `T`. + template<typename U, typename = detail::enable_if<is_convertible<U>()>> + expect(expect<U>&& src) noexcept(std::is_nothrow_constructible<T, U>()) + : code_(src.error()), storage_() + { + if (src.has_value()) + store(std::move(*src)); + } + + ~expect() noexcept + { + if (has_value()) + get().~T(); + } + + expect& operator=(expect const& src) noexcept(std::is_nothrow_copy_constructible<T>() && std::is_nothrow_copy_assignable<T>()) + { + if (this != std::addressof(src)) + { + if (has_value() && src.has_value()) + get() = src.get(); + else if (has_value()) + get().~T(); + else if (src.has_value()) + store(src.get()); + code_ = src.error(); + } + return *this; + } + + /*! Move `src` into `this`. If `src.has_value() && addressof(src) != this` + then `src.value() will be in a "moved from state". */ + expect& operator=(expect&& src) noexcept(std::is_nothrow_move_constructible<T>() && std::is_nothrow_move_assignable<T>()) + { + if (this != std::addressof(src)) + { + if (has_value() && src.has_value()) + get() = std::move(src.get()); + else if (has_value()) + get().~T(); + else if (src.has_value()) + store(std::move(src.get())); + code_ = src.error(); + } + return *this; + } + + //! \return True if `this` is storing a value instead of an error. + explicit operator bool() const noexcept { return has_value(); } + + //! \return True if `this` is storing an error instead of a value. + bool has_error() const noexcept { return bool(code_); } + + //! \return True if `this` is storing a value instead of an error. + bool has_value() const noexcept { return !has_error(); } + + //! \return Error - always safe to call. Empty when `!has_error()`. + std::error_code error() const noexcept { return code_; } + + //! \return Value if `has_value()` otherwise \throw `std::system_error{error()}`. + T& value() & + { + maybe_throw(); + return get(); + } + + //! \return Value if `has_value()` otherwise \throw `std::system_error{error()}`. + T const& value() const & + { + maybe_throw(); + return get(); + } + + /*! Same as other overloads, but expressions such as `foo(bar().value())` + will automatically perform moves with no copies. */ + T&& value() && + { + maybe_throw(); + return std::move(get()); + } + + //! \return Value, \pre `has_value()`. + T* operator->() noexcept { return std::addressof(get()); } + //! \return Value, \pre `has_value()`. + T const* operator->() const noexcept { return std::addressof(get()); } + //! \return Value, \pre `has_value()`. + T& operator*() noexcept { return get(); } + //! \return Value, \pre `has_value()`. + T const& operator*() const noexcept { return get(); } + + /*! + \note This function is `noexcept` when `U == T` is `noexcept`. + \return True if `has_value() == rhs.has_value()` and if values or errors are equal. + */ + template<typename U> + bool equal(expect<U> const& rhs) const noexcept(noexcept(*std::declval<expect<T>>() == *rhs)) + { + return has_value() && rhs.has_value() ? + get() == *rhs : error() == rhs.error(); + } + + //! \return False if `has_value()`, otherwise `error() == rhs`. + bool equal(std::error_code const& rhs) const noexcept + { + return has_error() && error() == rhs; + } + + /*! + \note This function is `noexcept` when `U == T` is `noexcept`. + \return False if `has_error()`, otherwise `value() == rhs`. + */ + template<typename U, typename = detail::enable_if<!std::is_constructible<std::error_code, U>::value>> + bool equal(U const& rhs) const noexcept(noexcept(*std::declval<expect<T>>() == rhs)) + { + return has_value() && get() == rhs; + } + + //! \return False if `has_value()`, otherwise `error() == rhs`. + bool matches(std::error_condition const& rhs) const noexcept + { + return has_error() && error() == rhs; + } +}; + +template<> +class expect<void> +{ + std::error_code code_; + +public: + using value_type = void; + using error_type = std::error_code; + + //! Create a successful object. + expect() noexcept + : code_() + {} + + expect(std::error_code const& code) noexcept + : code_(code) + { + if (!has_error()) + code_ = ::common_error::kInvalidErrorCode; + } + + expect(expect const&) = default; + ~expect() = default; + expect& operator=(expect const&) = default; + + //! \return True if `this` is storing a value instead of an error. + explicit operator bool() const noexcept { return !has_error(); } + + //! \return True if `this` is storing an error instead of a value. + bool has_error() const noexcept { return bool(code_); } + + //! \return Error - alway + std::error_code error() const noexcept { return code_; } + + //! \return `error() == rhs.error()`. + bool equal(expect const& rhs) const noexcept + { + return error() == rhs.error(); + } + + //! \return `has_error() && error() == rhs`. + bool equal(std::error_code const& rhs) const noexcept + { + return has_error() && error() == rhs; + } + + //! \return False if `has_value()`, otherwise `error() == rhs`. + bool matches(std::error_condition const& rhs) const noexcept + { + return has_error() && error() == rhs; + } +}; + +//! \return An `expect<void>` object with `!has_error()`. +inline expect<void> success() noexcept { return expect<void>{}; } + +template<typename T, typename U> +inline +bool operator==(expect<T> const& lhs, expect<U> const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return lhs.equal(rhs); +} + +template<typename T, typename U> +inline +bool operator==(expect<T> const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return lhs.equal(rhs); +} + +template<typename T, typename U> +inline +bool operator==(T const& lhs, expect<U> const& rhs) noexcept(noexcept(rhs.equal(lhs))) +{ + return rhs.equal(lhs); +} + +template<typename T, typename U> +inline +bool operator!=(expect<T> const& lhs, expect<U> const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return !lhs.equal(rhs); +} + +template<typename T, typename U> +inline +bool operator!=(expect<T> const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs))) +{ + return !lhs.equal(rhs); +} + +template<typename T, typename U> +inline +bool operator!=(T const& lhs, expect<U> const& rhs) noexcept(noexcept(rhs.equal(lhs))) +{ + return !rhs.equal(lhs); +} + +namespace detail +{ + inline void expect::unwrap(::expect<void>&& result, const char* error_msg, const char* file, unsigned line) + { + if (!result) + throw_(result.error(), error_msg, file, line); + } +} + diff --git a/src/common/json_util.h b/src/common/json_util.h index 661022a6f..c320c3956 100644 --- a/src/common/json_util.h +++ b/src/common/json_util.h @@ -29,14 +29,14 @@ #pragma once #define GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, name, type, jtype, mandatory, def) \ - type field_##name = def; \ + type field_##name = static_cast<type>(def); \ bool field_##name##_found = false; \ (void)field_##name##_found; \ do if (json.HasMember(#name)) \ { \ if (json[#name].Is##jtype()) \ { \ - field_##name = json[#name].Get##jtype(); \ + field_##name = static_cast<type>(json[#name].Get##jtype()); \ field_##name##_found = true; \ } \ else \ diff --git a/src/common/notify.cpp b/src/common/notify.cpp new file mode 100644 index 000000000..cadc68ea7 --- /dev/null +++ b/src/common/notify.cpp @@ -0,0 +1,62 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/algorithm/string.hpp> +#include "misc_log_ex.h" +#include "file_io_utils.h" +#include "spawn.h" +#include "notify.h" + +namespace tools +{ + +/* + TODO: + - Improve tokenization to handle paths containing whitespaces, quotes, etc. + - Windows unicode support (implies implementing unicode command line parsing code) +*/ +Notify::Notify(const char *spec) +{ + CHECK_AND_ASSERT_THROW_MES(spec, "Null spec"); + + boost::split(args, spec, boost::is_any_of(" ")); + CHECK_AND_ASSERT_THROW_MES(args.size() > 0, "Failed to parse spec"); + filename = args[0]; + CHECK_AND_ASSERT_THROW_MES(epee::file_io_utils::is_file_exist(filename), "File not found: " << filename); +} + +int Notify::notify(const char *parameter) +{ + std::vector<std::string> margs = args; + for (std::string &s: margs) + boost::replace_all(s, "%s", parameter); + + return tools::spawn(filename.c_str(), margs, false); +} + +} diff --git a/src/common/notify.h b/src/common/notify.h new file mode 100644 index 000000000..81aacebb0 --- /dev/null +++ b/src/common/notify.h @@ -0,0 +1,49 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <string> +#include <vector> + +namespace tools +{ + +class Notify +{ +public: + Notify(const char *spec); + + int notify(const char *parameter); + +private: + std::string filename; + std::vector<std::string> args; +}; + +} diff --git a/src/common/password.cpp b/src/common/password.cpp index 9336a14fc..b3c51128f 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -54,45 +54,61 @@ namespace return 0 != _isatty(_fileno(stdin)); } - bool read_from_tty(epee::wipeable_string& pass) + bool read_from_tty(epee::wipeable_string& pass, bool hide_input) { - static constexpr const char BACKSPACE = 8; - HANDLE h_cin = ::GetStdHandle(STD_INPUT_HANDLE); DWORD mode_old; ::GetConsoleMode(h_cin, &mode_old); - DWORD mode_new = mode_old & ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT); + DWORD mode_new = mode_old & ~(hide_input ? ENABLE_ECHO_INPUT : 0); ::SetConsoleMode(h_cin, mode_new); bool r = true; pass.reserve(tools::password_container::max_password_size); + std::vector<int> chlen; + chlen.reserve(tools::password_container::max_password_size); while (pass.size() < tools::password_container::max_password_size) { DWORD read; - char ch; - r = (TRUE == ::ReadConsoleA(h_cin, &ch, 1, &read, NULL)); + wchar_t ucs2_ch; + r = (TRUE == ::ReadConsoleW(h_cin, &ucs2_ch, 1, &read, NULL)); r &= (1 == read); + if (!r) { break; } - else if (ch == '\n' || ch == '\r') + else if (ucs2_ch == L'\r') + { + continue; + } + else if (ucs2_ch == L'\n') { std::cout << std::endl; break; } - else if (ch == BACKSPACE) + else if (ucs2_ch == L'\b') { if (!pass.empty()) { - pass.pop_back(); + int len = chlen.back(); + chlen.pop_back(); + while(len-- > 0) + pass.pop_back(); } + continue; } - else - { - pass.push_back(ch); - } + + char utf8_ch[8] = {0}; + int len; + if((len = WideCharToMultiByte(CP_UTF8, 0, &ucs2_ch, 1, utf8_ch, sizeof(utf8_ch), NULL, NULL)) <= 0) + break; + + if(pass.size() + len >= tools::password_container::max_password_size) + break; + + chlen.push_back(len); + pass += utf8_ch; } ::SetConsoleMode(h_cin, mode_old); @@ -107,14 +123,14 @@ namespace return 0 != isatty(fileno(stdin)); } - int getch() noexcept + int getch(bool hide_input) noexcept { struct termios tty_old; tcgetattr(STDIN_FILENO, &tty_old); struct termios tty_new; tty_new = tty_old; - tty_new.c_lflag &= ~(ICANON | ECHO); + tty_new.c_lflag &= ~(ICANON | (hide_input ? ECHO : 0)); tcsetattr(STDIN_FILENO, TCSANOW, &tty_new); int ch = getchar(); @@ -124,14 +140,14 @@ namespace return ch; } - bool read_from_tty(epee::wipeable_string& aPass) + bool read_from_tty(epee::wipeable_string& aPass, bool hide_input) { static constexpr const char BACKSPACE = 127; aPass.reserve(tools::password_container::max_password_size); while (aPass.size() < tools::password_container::max_password_size) { - int ch = getch(); + int ch = getch(hide_input); if (EOF == ch || ch == EOT) { return false; @@ -146,6 +162,13 @@ namespace if (!aPass.empty()) { aPass.pop_back(); + if (!hide_input) + std::cout << "\b\b\b \b\b\b" << std::flush; + } + else + { + if (!hide_input) + std::cout << "\b\b \b\b" << std::flush; } } else @@ -159,18 +182,18 @@ namespace #endif // end !WIN32 - bool read_from_tty(const bool verify, const char *message, epee::wipeable_string& pass1, epee::wipeable_string& pass2) + bool read_from_tty(const bool verify, const char *message, bool hide_input, epee::wipeable_string& pass1, epee::wipeable_string& pass2) { while (true) { if (message) - std::cout << message <<": "; - if (!read_from_tty(pass1)) + std::cout << message <<": " << std::flush; + if (!read_from_tty(pass1, hide_input)) return false; if (verify) { std::cout << "Confirm password: "; - if (!read_from_tty(pass2)) + if (!read_from_tty(pass2, hide_input)) return false; if(pass1!=pass2) { @@ -221,6 +244,10 @@ namespace tools : m_password(std::move(password)) { } + password_container::password_container(const epee::wipeable_string& password) noexcept + : m_password(password) + { + } password_container::~password_container() noexcept { @@ -229,12 +256,12 @@ namespace tools std::atomic<bool> password_container::is_prompting(false); - boost::optional<password_container> password_container::prompt(const bool verify, const char *message) + boost::optional<password_container> password_container::prompt(const bool verify, const char *message, bool hide_input) { is_prompting = true; password_container pass1{}; password_container pass2{}; - if (is_cin_tty() ? read_from_tty(verify, message, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) + if (is_cin_tty() ? read_from_tty(verify, message, hide_input, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) { is_prompting = false; return {std::move(pass1)}; diff --git a/src/common/password.h b/src/common/password.h index 61937b93a..beb98283b 100644 --- a/src/common/password.h +++ b/src/common/password.h @@ -47,9 +47,10 @@ namespace tools //! `password` is used as password password_container(std::string&& password) noexcept; + password_container(const epee::wipeable_string& password) noexcept; //! \return A password from stdin TTY prompt or `std::cin` pipe. - static boost::optional<password_container> prompt(bool verify, const char *mesage = "Password"); + static boost::optional<password_container> prompt(bool verify, const char *mesage = "Password", bool hide_input = true); static std::atomic<bool> is_prompting; password_container(const password_container&) = delete; diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 16abdfd99..47f01de65 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -33,7 +33,7 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "perf" -namespace +namespace tools { uint64_t get_tick_count() { @@ -83,7 +83,7 @@ namespace tools el::Level performance_timer_log_level = el::Level::Debug; -static __thread std::vector<PerformanceTimer*> *performance_timers = NULL; +static __thread std::vector<LoggingPerformanceTimer*> *performance_timers = NULL; void set_performance_timer_log_level(el::Level level) { @@ -96,21 +96,28 @@ void set_performance_timer_log_level(el::Level level) performance_timer_log_level = level; } -PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Level l): name(s), unit(unit), level(l), started(false), paused(false) +PerformanceTimer::PerformanceTimer(bool paused): started(true), paused(paused) +{ + if (paused) + ticks = 0; + else + ticks = get_tick_count(); +} + +LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l): PerformanceTimer(), name(s), cat(cat), unit(unit), level(l) { - ticks = get_tick_count(); if (!performance_timers) { - MLOG(level, "PERF ----------"); - performance_timers = new std::vector<PerformanceTimer*>(); + MCLOG(level, cat.c_str(), "PERF ----------"); + performance_timers = new std::vector<LoggingPerformanceTimer*>(); } else { - PerformanceTimer *pt = performance_timers->back(); + LoggingPerformanceTimer *pt = performance_timers->back(); if (!pt->started && !pt->paused) { size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; - MLOG(pt->level, "PERF " << std::string((size-1) * 2, ' ') << " " << pt->name); + MCLOG(pt->level, cat.c_str(), "PERF " << std::string((size-1) * 2, ' ') << " " << pt->name); pt->started = true; } } @@ -119,13 +126,18 @@ PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Leve PerformanceTimer::~PerformanceTimer() { - performance_timers->pop_back(); if (!paused) ticks = get_tick_count() - ticks; +} + +LoggingPerformanceTimer::~LoggingPerformanceTimer() +{ + pause(); + performance_timers->pop_back(); char s[12]; snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; - MLOG(level, "PERF " << s << std::string(size * 2, ' ') << " " << name); + MCLOG(level, cat.c_str(), "PERF " << s << std::string(size * 2, ' ') << " " << name); if (performance_timers->empty()) { delete performance_timers; diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index 0e910caf9..1d4dee5b5 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -33,9 +33,6 @@ #include <memory> #include "misc_log_ex.h" -#undef MONERO_DEFAULT_LOG_CATEGORY -#define MONERO_DEFAULT_LOG_CATEGORY "perf" - namespace tools { @@ -43,31 +40,48 @@ class PerformanceTimer; extern el::Level performance_timer_log_level; +uint64_t get_tick_count(); +uint64_t get_ticks_per_ns(); +uint64_t ticks_to_ns(uint64_t ticks); + class PerformanceTimer { public: - PerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + PerformanceTimer(bool paused = false); ~PerformanceTimer(); void pause(); void resume(); + uint64_t value() const { return ticks; } +void set(uint64_t v){ticks=v;} + +protected: + uint64_t ticks; + bool started; + bool paused; +}; + +class LoggingPerformanceTimer: public PerformanceTimer +{ +public: + LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l = el::Level::Debug); + ~LoggingPerformanceTimer(); + private: std::string name; + std::string cat; uint64_t unit; el::Level level; - uint64_t ticks; - bool started; - bool paused; }; void set_performance_timer_log_level(el::Level level); -#define PERF_TIMER_UNIT(name, unit) tools::PerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) -#define PERF_TIMER_UNIT_L(name, unit, l) tools::PerformanceTimer pt_##name(#name, unit, l) -#define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000) -#define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000, l) -#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::PerformanceTimer> pt_##name(new tools::PerformanceTimer(#name, unit, el::Level::Info)) -#define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000) +#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, tools::performance_timer_log_level) +#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, l) +#define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000000) +#define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000000, l) +#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> pt_##name(new tools::LoggingPerformanceTimer(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, el::Level::Info)) +#define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000000) #define PERF_TIMER_STOP(name) do { pt_##name.reset(NULL); } while(0) #define PERF_TIMER_PAUSE(name) pt_##name->pause() #define PERF_TIMER_RESUME(name) pt_##name->resume() diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index d7517babb..d887a13c9 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -73,7 +73,7 @@ public: #if defined(_MSC_VER) , m_oss(std::move(rhs.m_oss)) #else - // GCC bug: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 + // GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 , m_oss(rhs.m_oss.str(), std::ios_base::out | std::ios_base::ate) #endif , m_color(std::move(rhs.m_color)) diff --git a/src/common/spawn.cpp b/src/common/spawn.cpp new file mode 100644 index 000000000..0a2ce8387 --- /dev/null +++ b/src/common/spawn.cpp @@ -0,0 +1,144 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <errno.h> +#include <unistd.h> +#include <sys/types.h> +#ifdef _WIN32 +#include <boost/algorithm/string/join.hpp> +#include <boost/scope_exit.hpp> +#include <windows.h> +#else +#include <sys/wait.h> +#endif + +#include "misc_log_ex.h" +#include "util.h" +#include "spawn.h" + +namespace tools +{ + +int spawn(const char *filename, const std::vector<std::string>& args, bool wait) +{ +#ifdef _WIN32 + std::string joined = boost::algorithm::join(args, " "); + char *commandLine = !joined.empty() ? &joined[0] : nullptr; + STARTUPINFOA si = {}; + si.cb = sizeof(si); + PROCESS_INFORMATION pi; + if (!CreateProcessA(filename, commandLine, nullptr, nullptr, false, 0, nullptr, nullptr, &si, &pi)) + { + MERROR("CreateProcess failed. Error code " << GetLastError()); + return -1; + } + + BOOST_SCOPE_EXIT(&pi) + { + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + } + BOOST_SCOPE_EXIT_END + + if (!wait) + { + return 0; + } + + DWORD result = WaitForSingleObject(pi.hProcess, INFINITE); + if (result != WAIT_OBJECT_0) + { + MERROR("WaitForSingleObject failed. Result " << result << ", error code " << GetLastError()); + return -1; + } + + DWORD exitCode; + if (!GetExitCodeProcess(pi.hProcess, &exitCode)) + { + MERROR("GetExitCodeProcess failed. Error code " << GetLastError()); + return -1; + } + + MINFO("Child exited with " << exitCode); + return static_cast<int>(exitCode); +#else + char **argv = (char**)alloca(sizeof(char*) * (args.size() + 1)); + for (size_t n = 0; n < args.size(); ++n) + argv[n] = (char*)args[n].c_str(); + argv[args.size()] = NULL; + + pid_t pid = fork(); + if (pid < 0) + { + MERROR("Error forking: " << strerror(errno)); + return -1; + } + + // child + if (pid == 0) + { + tools::closefrom(3); + close(0); + char *envp[] = {NULL}; + execve(filename, argv, envp); + MERROR("Failed to execve: " << strerror(errno)); + return -1; + } + + // parent + if (pid > 0) + { + if (!wait) + return 0; + + while (1) + { + int wstatus = 0; + pid_t w = waitpid(pid, &wstatus, WUNTRACED | WCONTINUED); + if (w < 0) { + MERROR("Error waiting for child: " << strerror(errno)); + return -1; + } + if (WIFEXITED(wstatus)) + { + MINFO("Child exited with " << WEXITSTATUS(wstatus)); + return WEXITSTATUS(wstatus); + } + if (WIFSIGNALED(wstatus)) + { + MINFO("Child killed by " << WEXITSTATUS(wstatus)); + return WEXITSTATUS(wstatus); + } + } + } + MERROR("Secret passage found"); + return -1; +#endif +} + +} diff --git a/src/common/spawn.h b/src/common/spawn.h new file mode 100644 index 000000000..c90a0f790 --- /dev/null +++ b/src/common/spawn.h @@ -0,0 +1,36 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace tools +{ + +int spawn(const char *filename, const std::vector<std::string>& args, bool wait); + +} diff --git a/src/common/stack_trace.cpp b/src/common/stack_trace.cpp index 9c2bf4b53..141621427 100644 --- a/src/common/stack_trace.cpp +++ b/src/common/stack_trace.cpp @@ -49,9 +49,18 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "stacktrace" -#define ST_LOG(x) CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,MONERO_DEFAULT_LOG_CATEGORY) << x - -// from http://stackoverflow.com/questions/11665829/how-can-i-print-stack-trace-for-caught-exceptions-in-c-code-injection-in-c +#define ST_LOG(x) \ + do { \ + auto elpp = ELPP; \ + if (elpp) { \ + CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,MONERO_DEFAULT_LOG_CATEGORY) << x; \ + } \ + else { \ + std::cout << x << std::endl; \ + } \ + } while(0) + +// from https://stackoverflow.com/questions/11665829/how-can-i-print-stack-trace-for-caught-exceptions-in-c-code-injection-in-c // The decl of __cxa_throw in /usr/include/.../cxxabi.h uses // 'std::type_info *', but GCC's built-in protype uses 'void *'. diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 51e071577..37825e31d 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -36,62 +36,79 @@ #include "common/util.h" static __thread int depth = 0; +static __thread bool is_leaf = false; namespace tools { -threadpool::threadpool() : running(true), active(0) { +threadpool::threadpool(unsigned int max_threads) : running(true), active(0) { boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - max = tools::get_max_concurrency(); - size_t i = max; + max = max_threads ? max_threads : tools::get_max_concurrency(); + size_t i = max ? max - 1 : 0; while(i--) { - threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this))); + threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false))); } } threadpool::~threadpool() { + try { const boost::unique_lock<boost::mutex> lock(mutex); running = false; has_work.notify_all(); } + catch (...) + { + // if the lock throws, we're just do it without a lock and hope, + // since the alternative is terminate + running = false; + has_work.notify_all(); + } for (size_t i = 0; i<threads.size(); i++) { - threads[i].join(); + try { threads[i].join(); } + catch (...) { /* ignore */ } } } -void threadpool::submit(waiter *obj, std::function<void()> f) { - entry e = {obj, f}; +void threadpool::submit(waiter *obj, std::function<void()> f, bool leaf) { + CHECK_AND_ASSERT_THROW_MES(!is_leaf, "A leaf routine is using a thread pool"); boost::unique_lock<boost::mutex> lock(mutex); - if ((active == max && !queue.empty()) || depth > 0) { + if (!leaf && ((active == max && !queue.empty()) || depth > 0)) { // if all available threads are already running // and there's work waiting, just run in current thread lock.unlock(); ++depth; + is_leaf = leaf; f(); --depth; + is_leaf = false; } else { if (obj) obj->inc(); - queue.push_back(e); + if (leaf) + queue.push_front({obj, f, leaf}); + else + queue.push_back({obj, f, leaf}); has_work.notify_one(); } } -int threadpool::get_max_concurrency() { +unsigned int threadpool::get_max_concurrency() const { return max; } threadpool::waiter::~waiter() { + try { boost::unique_lock<boost::mutex> lock(mt); if (num) MERROR("wait should have been called before waiter dtor - waiting now"); } + catch (...) { /* ignore */ } try { - wait(); + wait(NULL); } catch (const std::exception &e) { @@ -99,9 +116,12 @@ threadpool::waiter::~waiter() } } -void threadpool::waiter::wait() { +void threadpool::waiter::wait(threadpool *tpool) { + if (tpool) + tpool->run(true); boost::unique_lock<boost::mutex> lock(mt); - while(num) cv.wait(lock); + while(num) + cv.wait(lock); } void threadpool::waiter::inc() { @@ -113,15 +133,19 @@ void threadpool::waiter::dec() { const boost::unique_lock<boost::mutex> lock(mt); num--; if (!num) - cv.notify_one(); + cv.notify_all(); } -void threadpool::run() { +void threadpool::run(bool flush) { boost::unique_lock<boost::mutex> lock(mutex); while (running) { entry e; while(queue.empty() && running) + { + if (flush) + return; has_work.wait(lock); + } if (!running) break; active++; @@ -129,8 +153,10 @@ void threadpool::run() { queue.pop_front(); lock.unlock(); ++depth; + is_leaf = e.leaf; e.f(); --depth; + is_leaf = false; if (e.wo) e.wo->dec(); diff --git a/src/common/threadpool.h b/src/common/threadpool.h index 34152541c..a43e38a76 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -46,6 +46,9 @@ public: static threadpool instance; return instance; } + static threadpool *getNewForUnitTests(unsigned max_threads = 0) { + return new threadpool(max_threads); + } // The waiter lets the caller know when all of its // tasks are completed. @@ -56,7 +59,7 @@ public: public: void inc(); void dec(); - void wait(); //! Wait for a set of tasks to finish. + void wait(threadpool *tpool); //! Wait for a set of tasks to finish. waiter() : num(0){} ~waiter(); }; @@ -64,25 +67,27 @@ public: // Submit a task to the pool. The waiter pointer may be // NULL if the caller doesn't care to wait for the // task to finish. - void submit(waiter *waiter, std::function<void()> f); + void submit(waiter *waiter, std::function<void()> f, bool leaf = false); + + unsigned int get_max_concurrency() const; - int get_max_concurrency(); + ~threadpool(); private: - threadpool(); - ~threadpool(); + threadpool(unsigned int max_threads = 0); typedef struct entry { waiter *wo; std::function<void()> f; + bool leaf; } entry; std::deque<entry> queue; boost::condition_variable has_work; boost::mutex mutex; std::vector<boost::thread> threads; - int active; - int max; + unsigned int active; + unsigned int max; bool running; - void run(); + void run(bool flush = false); }; } diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 9eb402e0b..9f12f8dbc 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -69,12 +69,12 @@ namespace tools continue; bool alnum = true; - for (auto c: hash) + for (auto c: fields[3]) if (!isalnum(c)) alnum = false; - if (hash.size() != 64 && !alnum) + if (fields[3].size() != 64 && !alnum) { - MWARNING("Invalid hash: " << hash); + MWARNING("Invalid hash: " << fields[3]); continue; } diff --git a/src/common/util.cpp b/src/common/util.cpp index 008610117..43973c511 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -28,6 +28,7 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#include <unistd.h> #include <cstdio> #ifdef __GLIBC__ @@ -37,7 +38,7 @@ #ifdef __GLIBC__ #include <sys/types.h> #include <sys/stat.h> -#include <ustat.h> +#include <sys/resource.h> #include <unistd.h> #include <dirent.h> #include <string.h> @@ -45,6 +46,13 @@ #include <string> #endif +//tools::is_hdd +#ifdef __GLIBC__ + #include <sstream> + #include <sys/sysmacros.h> + #include <fstream> +#endif + #include "unbound.h" #include "include_base_utils.h" @@ -195,6 +203,73 @@ namespace tools catch (...) {} } + file_locker::file_locker(const std::string &filename) + { +#ifdef WIN32 + m_fd = INVALID_HANDLE_VALUE; + std::wstring filename_wide; + try + { + filename_wide = string_tools::utf8_to_utf16(filename); + } + catch (const std::exception &e) + { + MERROR("Failed to convert path \"" << filename << "\" to UTF-16: " << e.what()); + return; + } + m_fd = CreateFileW(filename_wide.c_str(), GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (m_fd != INVALID_HANDLE_VALUE) + { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(m_fd, LOCKFILE_FAIL_IMMEDIATELY | LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ov)) + { + MERROR("Failed to lock " << filename << ": " << std::error_code(GetLastError(), std::system_category())); + CloseHandle(m_fd); + m_fd = INVALID_HANDLE_VALUE; + } + } + else + { + MERROR("Failed to open " << filename << ": " << std::error_code(GetLastError(), std::system_category())); + } +#else + m_fd = open(filename.c_str(), O_RDONLY | O_CREAT | O_CLOEXEC, 0666); + if (m_fd != -1) + { + if (flock(m_fd, LOCK_EX | LOCK_NB) == -1) + { + MERROR("Failed to lock " << filename << ": " << std::strerror(errno)); + close(m_fd); + m_fd = -1; + } + } + else + { + MERROR("Failed to open " << filename << ": " << std::strerror(errno)); + } +#endif + } + file_locker::~file_locker() + { + if (locked()) + { +#ifdef WIN32 + CloseHandle(m_fd); +#else + close(m_fd); +#endif + } + } + bool file_locker::locked() const + { +#ifdef WIN32 + return m_fd != INVALID_HANDLE_VALUE; +#else + return m_fd != -1; +#endif + } + #ifdef WIN32 std::string get_windows_version_display_string() { @@ -233,10 +308,19 @@ namespace tools StringCchCopy(pszOS, BUFSIZE, TEXT("Microsoft ")); // Test for the specific product. + if ( osvi.dwMajorVersion == 10 ) + { + if ( osvi.dwMinorVersion == 0 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 10 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2016 " )); + } + } if ( osvi.dwMajorVersion == 6 ) { - if( osvi.dwMinorVersion == 0 ) + if ( osvi.dwMinorVersion == 0 ) { if( osvi.wProductType == VER_NT_WORKSTATION ) StringCchCat(pszOS, BUFSIZE, TEXT("Windows Vista ")); @@ -250,6 +334,20 @@ namespace tools else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2008 R2 " )); } + if ( osvi.dwMinorVersion == 2 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 8 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2012 " )); + } + + if ( osvi.dwMinorVersion == 3 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 8.1 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2012 R2 " )); + } + pGPI = (PGPI) GetProcAddress( GetModuleHandle(TEXT("kernel32.dll")), "GetProductInfo"); @@ -451,10 +549,15 @@ std::string get_nix_version_display_string() if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate)) { - int size_needed = WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), NULL, 0, NULL, NULL); - std::string folder_name(size_needed, 0); - WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), &folder_name[0], size_needed, NULL, NULL); - return folder_name; + try + { + return string_tools::utf16_to_utf8(psz_path); + } + catch (const std::exception &e) + { + MERROR("utf16_to_utf8 failed: " << e.what()); + return ""; + } } LOG_ERROR("SHGetSpecialFolderPathW() failed, could not obtain requested path."); @@ -515,18 +618,20 @@ std::string get_nix_version_display_string() int code; #if defined(WIN32) // Maximizing chances for success - WCHAR wide_replacement_name[1000]; - MultiByteToWideChar(CP_UTF8, 0, replacement_name.c_str(), replacement_name.size() + 1, wide_replacement_name, 1000); - WCHAR wide_replaced_name[1000]; - MultiByteToWideChar(CP_UTF8, 0, replaced_name.c_str(), replaced_name.size() + 1, wide_replaced_name, 1000); - - DWORD attributes = ::GetFileAttributesW(wide_replaced_name); + std::wstring wide_replacement_name; + try { wide_replacement_name = string_tools::utf8_to_utf16(replacement_name); } + catch (...) { return std::error_code(GetLastError(), std::system_category()); } + std::wstring wide_replaced_name; + try { wide_replaced_name = string_tools::utf8_to_utf16(replaced_name); } + catch (...) { return std::error_code(GetLastError(), std::system_category()); } + + DWORD attributes = ::GetFileAttributesW(wide_replaced_name.c_str()); if (INVALID_FILE_ATTRIBUTES != attributes) { - ::SetFileAttributesW(wide_replaced_name, attributes & (~FILE_ATTRIBUTE_READONLY)); + ::SetFileAttributesW(wide_replaced_name.c_str(), attributes & (~FILE_ATTRIBUTE_READONLY)); } - bool ok = 0 != ::MoveFileExW(wide_replacement_name, wide_replaced_name, MOVEFILE_REPLACE_EXISTING); + bool ok = 0 != ::MoveFileExW(wide_replacement_name.c_str(), wide_replaced_name.c_str(), MOVEFILE_REPLACE_EXISTING); code = ok ? 0 : static_cast<int>(::GetLastError()); #else bool ok = 0 == std::rename(replacement_name.c_str(), replaced_name.c_str()); @@ -608,6 +713,21 @@ std::string get_nix_version_display_string() static void setup_crash_dump() {} #endif + bool disable_core_dumps() + { +#ifdef __GLIBC__ + // disable core dumps in release mode + struct rlimit rlimit; + rlimit.rlim_cur = rlimit.rlim_max = 0; + if (setrlimit(RLIMIT_CORE, &rlimit)) + { + MWARNING("Failed to disable core dumps"); + return false; + } +#endif + return true; + } + bool on_startup() { mlog_configure("", true); @@ -643,62 +763,41 @@ std::string get_nix_version_display_string() #endif } - bool is_hdd(const char *path) + boost::optional<bool> is_hdd(const char *file_path) { #ifdef __GLIBC__ - std::string device = ""; - struct stat st, dst; - if (stat(path, &st) < 0) - return 0; - - DIR *dir = opendir("/dev/block"); - if (!dir) - return 0; - struct dirent *de; - while ((de = readdir(dir))) - { - if (strcmp(de->d_name, ".") && strcmp(de->d_name, "..")) + struct stat st; + std::string prefix; + if(stat(file_path, &st) == 0) + { + std::ostringstream s; + s << "/sys/dev/block/" << major(st.st_dev) << ":" << minor(st.st_dev); + prefix = s.str(); + } + else + { + return boost::none; + } + std::string attr_path = prefix + "/queue/rotational"; + std::ifstream f(attr_path, std::ios_base::in); + if(not f.is_open()) + { + attr_path = prefix + "/../queue/rotational"; + f.open(attr_path, std::ios_base::in); + if(not f.is_open()) { - std::string dev_path = std::string("/dev/block/") + de->d_name; - char resolved[PATH_MAX]; - if (realpath(dev_path.c_str(), resolved) && !strncmp(resolved, "/dev/", 5)) - { - if (stat(resolved, &dst) == 0) - { - if (dst.st_rdev == st.st_dev) - { - // take out trailing digits (eg, sda1 -> sda) - char *ptr = resolved; - while (*ptr) - ++ptr; - while (ptr > resolved && isdigit(*--ptr)) - *ptr = 0; - device = resolved + 5; - break; - } - } - } + return boost::none; } } - closedir(dir); - - if (device.empty()) - return 0; - - std::string sys_path = "/sys/block/" + device + "/queue/rotational"; - FILE *f = fopen(sys_path.c_str(), "r"); - if (!f) - return false; - char s[8]; - char *ptr = fgets(s, sizeof(s), f); - fclose(f); - if (!ptr) - return 0; - s[sizeof(s) - 1] = 0; - int n = atoi(s); // returns 0 on parse error - return n == 1; + unsigned short val = 0xdead; + f >> val; + if(not f.fail()) + { + return (val == 1); + } + return boost::none; #else - return 0; + return boost::none; #endif } @@ -727,6 +826,13 @@ std::string get_nix_version_display_string() bool is_local_address(const std::string &address) { + // always assume Tor/I2P addresses to be untrusted by default + if (boost::ends_with(address, ".onion") || boost::ends_with(address, ".i2p")) + { + MDEBUG("Address '" << address << "' is Tor/I2P, non local"); + return false; + } + // extract host epee::net_utils::http::url_content u_c; if (!epee::net_utils::parse_url(address, u_c)) @@ -820,4 +926,88 @@ std::string get_nix_version_display_string() return false; return true; } + + boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str) + { + auto pos = str.find(":"); + bool r = pos != std::string::npos; + uint32_t major; + r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos)); + uint32_t minor; + r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1)); + if (r) + { + return std::make_pair(major, minor); + } + else + { + return {}; + } + } + + std::string glob_to_regex(const std::string &val) + { + std::string newval; + + bool escape = false; + for (char c: val) + { + if (c == '*') + newval += escape ? "*" : ".*"; + else if (c == '?') + newval += escape ? "?" : "."; + else if (c == '\\') + newval += '\\', escape = !escape; + else + newval += c; + } + return newval; + } + +#ifdef _WIN32 + std::string input_line_win() + { + HANDLE hConIn = CreateFileW(L"CONIN$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr); + DWORD oldMode; + + FlushConsoleInputBuffer(hConIn); + GetConsoleMode(hConIn, &oldMode); + SetConsoleMode(hConIn, ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT); + + wchar_t buffer[1024]; + DWORD read; + + ReadConsoleW(hConIn, buffer, sizeof(buffer)/sizeof(wchar_t)-1, &read, nullptr); + buffer[read] = 0; + + SetConsoleMode(hConIn, oldMode); + CloseHandle(hConIn); + + int size_needed = WideCharToMultiByte(CP_UTF8, 0, buffer, -1, NULL, 0, NULL, NULL); + std::string buf(size_needed, '\0'); + WideCharToMultiByte(CP_UTF8, 0, buffer, -1, &buf[0], size_needed, NULL, NULL); + buf.pop_back(); //size_needed includes null that we needed to have space for + return buf; + } +#endif + + void closefrom(int fd) + { +#if defined __FreeBSD__ || defined __OpenBSD__ || defined __NetBSD__ || defined __DragonFly__ + ::closefrom(fd); +#else +#if defined __GLIBC__ + const int sc_open_max = sysconf(_SC_OPEN_MAX); + const int MAX_FDS = std::min(65536, sc_open_max); +#else + const int MAX_FDS = 65536; +#endif + while (fd < MAX_FDS) + { + close(fd); + ++fd; + } +#endif + } + } diff --git a/src/common/util.h b/src/common/util.h index 7caf0e3c5..e793a42b5 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -32,6 +32,7 @@ #include <boost/thread/locks.hpp> #include <boost/thread/mutex.hpp> +#include <boost/optional.hpp> #include <system_error> #include <csignal> #include <cstdio> @@ -90,6 +91,20 @@ namespace tools const std::string& filename() const noexcept { return m_filename; } }; + class file_locker + { + public: + file_locker(const std::string &filename); + ~file_locker(); + bool locked() const; + private: +#ifdef WIN32 + HANDLE m_fd; +#else + int m_fd; +#endif + }; + /*! \brief Returns the default data directory. * * \details Windows < Vista: C:\\Documents and Settings\\Username\\Application Data\\CRYPTONOTE_NAME @@ -134,6 +149,8 @@ namespace tools bool sanitize_locale(); + bool disable_core_dumps(); + bool on_startup(); /*! \brief Defines a signal handler for win32 and *nix @@ -213,5 +230,14 @@ namespace tools bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash); bool sha256sum(const std::string &filename, crypto::hash &hash); - bool is_hdd(const char *path); + boost::optional<bool> is_hdd(const char *path); + + boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str); + + std::string glob_to_regex(const std::string &val); +#ifdef _WIN32 + std::string input_line_win(); +#endif + + void closefrom(int fd); } diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 71dcedcab..0c635e7cb 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -78,6 +78,7 @@ target_link_libraries(cncrypto PUBLIC epee ${Boost_SYSTEM_LIBRARY} + ${SODIUM_LIBRARY} PRIVATE ${EXTRA_LIBRARIES}) diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index d503c47e0..6ef7d4207 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -31,7 +31,7 @@ * The blake256_* and blake224_* functions are largely copied from * blake256_light.c and blake224_light.c from the BLAKE website: * - * http://131002.net/blake/ + * https://131002.net/blake/ * * The hmac_* functions implement HMAC-BLAKE-256 and HMAC-BLAKE-224. * HMAC is specified by RFC 2104. diff --git a/src/crypto/chacha.h b/src/crypto/chacha.h index 2b3ed8043..6e85ad0e9 100644 --- a/src/crypto/chacha.h +++ b/src/crypto/chacha.h @@ -40,6 +40,7 @@ #include <memory.h> #include "memwipe.h" +#include "mlocker.h" #include "hash.h" namespace crypto { @@ -50,7 +51,7 @@ namespace crypto { #if defined(__cplusplus) } - using chacha_key = tools::scrubbed_arr<uint8_t, CHACHA_KEY_SIZE>; + using chacha_key = epee::mlocked<tools::scrubbed_arr<uint8_t, CHACHA_KEY_SIZE>>; #pragma pack(push, 1) // MS VC 2012 doesn't interpret `class chacha_iv` as POD in spite of [9.0.10], so it is a struct @@ -69,22 +70,26 @@ namespace crypto { chacha20(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher); } - inline void generate_chacha_key(const void *data, size_t size, chacha_key& key) { + inline void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr<char, HASH_SIZE> pwd_hash; + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + for (uint64_t n = 1; n < kdf_rounds; ++n) + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } - inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key) { + inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr<char, HASH_SIZE> pwd_hash; + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 1/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + for (uint64_t n = 1; n < kdf_rounds; ++n) + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } - inline void generate_chacha_key(std::string password, chacha_key& key) { - return generate_chacha_key(password.data(), password.size(), key); + inline void generate_chacha_key(std::string password, chacha_key& key, uint64_t kdf_rounds) { + return generate_chacha_key(password.data(), password.size(), key, kdf_rounds); } } diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 4ff4310de..1f77513ca 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -871,3 +871,9 @@ const fe fe_fffb2 = {8166131, -6741800, -17040804, 3154616, 21461005, 1466302, - const fe fe_fffb3 = {-13620103, 14639558, 4532995, 7679154, 16815101, -15883539, -22863840, -14813421, 13716513, -6477756}; /* sqrt(-sqrt(-1) * A * (A + 2)) */ const fe fe_fffb4 = {-21786234, -12173074, 21573800, 4524538, -4645904, 16204591, 8012863, -8444712, 3212926, 6885324}; /* sqrt(sqrt(-1) * A * (A + 2)) */ const ge_p3 ge_p3_identity = { {0}, {1, 0}, {1, 0}, {0} }; +const ge_p3 ge_p3_H = { + {7329926, -15101362, 31411471, 7614783, 27996851, -3197071, -11157635, -6878293, 466949, -7986503}, + {5858699, 5096796, 21321203, -7536921, -5553480, -11439507, -5627669, 15045946, 19977121, 5275251}, + {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {23443568, -5110398, -8776029, -4345135, 6889568, -14710814, 7474843, 3279062, 14550766, -7453428} +}; diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 45d412ac6..09296d6f9 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -3707,9 +3707,8 @@ void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, s[31] = s11 >> 17; } -/* Assumes that a != INT64_MIN */ static int64_t signum(int64_t a) { - return (a >> 63) - ((-a) >> 63); + return a > 0 ? 1 : a < 0 ? -1 : 0; } int sc_check(const unsigned char *s) { @@ -3730,3 +3729,16 @@ int sc_isnonzero(const unsigned char *s) { s[18] | s[19] | s[20] | s[21] | s[22] | s[23] | s[24] | s[25] | s[26] | s[27] | s[28] | s[29] | s[30] | s[31]) - 1) >> 8) + 1; } + +int ge_p3_is_point_at_infinity(const ge_p3 *p) { + // X = 0 and Y == Z + int n; + for (n = 0; n < 10; ++n) + { + if (p->X[n] | p->T[n]) + return 0; + if (p->Y[n] != p->Z[n]) + return 0; + } + return 1; +} diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index dc3c60794..2910dafd4 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -140,6 +140,7 @@ extern const fe fe_fffb2; extern const fe fe_fffb3; extern const fe fe_fffb4; extern const ge_p3 ge_p3_identity; +extern const ge_p3 ge_p3_H; void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); void sc_0(unsigned char *); void sc_reduce32(unsigned char *); @@ -158,3 +159,5 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); + +int ge_p3_is_point_at_infinity(const ge_p3 *p); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index f4ef751d3..ad7721cf0 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -70,6 +70,9 @@ namespace crypto { #include "random.h" } + const crypto::public_key null_pkey = crypto::public_key{}; + const crypto::secret_key null_skey = crypto::secret_key{}; + static inline unsigned char *operator &(ec_point &point) { return &reinterpret_cast<unsigned char &>(point); } @@ -93,18 +96,32 @@ namespace crypto { generate_random_bytes_not_thread_safe(N, bytes); } - /* generate a random 32-byte (256-bit) integer and copy it to res */ - static inline void random_scalar_not_thread_safe(ec_scalar &res) { - unsigned char tmp[64]; - generate_random_bytes_not_thread_safe(64, tmp); - sc_reduce(tmp); - memcpy(&res, tmp, 32); + static inline bool less32(const unsigned char *k0, const unsigned char *k1) + { + for (int n = 31; n >= 0; --n) + { + if (k0[n] < k1[n]) + return true; + if (k0[n] > k1[n]) + return false; + } + return false; } + + void random32_unbiased(unsigned char *bytes) + { + // l = 2^252 + 27742317777372353535851937790883648493. + // it fits 15 in 32 bytes + static const unsigned char limit[32] = { 0xe3, 0x6a, 0x67, 0x72, 0x8b, 0xce, 0x13, 0x29, 0x8f, 0x30, 0x82, 0x8c, 0x0b, 0xa4, 0x10, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0 }; + do + { + generate_random_bytes_thread_safe(32, bytes); + } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time + sc_reduce32(bytes); + } + /* generate a random 32-byte (256-bit) integer and copy it to res */ static inline void random_scalar(ec_scalar &res) { - unsigned char tmp[64]; - generate_random_bytes_thread_safe(64, tmp); - sc_reduce(tmp); - memcpy(&res, tmp, 32); + random32_unbiased((unsigned char*)res.data); } void hash_to_scalar(const void *data, size_t length, ec_scalar &res) { @@ -257,11 +274,18 @@ namespace crypto { #endif buf.h = prefix_hash; buf.key = pub; + try_again: random_scalar(k); + if (((const uint32_t*)(&k))[7] == 0) // we don't want tiny numbers here + goto try_again; ge_scalarmult_base(&tmp3, &k); ge_p3_tobytes(&buf.comm, &tmp3); hash_to_scalar(&buf, sizeof(s_comm), sig.c); + if (!sc_isnonzero((const unsigned char*)sig.c.data)) + goto try_again; sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k); + if (!sc_isnonzero((const unsigned char*)sig.r.data)) + goto try_again; } bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) { @@ -275,11 +299,14 @@ namespace crypto { if (ge_frombytes_vartime(&tmp3, &pub) != 0) { return false; } - if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0) { + if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0 || !sc_isnonzero(&sig.c)) { return false; } ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r); ge_tobytes(&buf.comm, &tmp2); + static const ec_point infinity = {{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + if (memcmp(&buf.comm, &infinity, 32) == 0) + return false; hash_to_scalar(&buf, sizeof(s_comm), c); sc_sub(&c, &c, &sig.c); return sc_isnonzero(&c) == 0; diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 9ea0f2ec0..33cc0a25a 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -34,7 +34,6 @@ #include <iostream> #include <boost/thread/mutex.hpp> #include <boost/thread/lock_guard.hpp> -#include <boost/utility/value_init.hpp> #include <boost/optional.hpp> #include <type_traits> #include <vector> @@ -42,6 +41,7 @@ #include "common/pod-class.h" #include "common/util.h" #include "memwipe.h" +#include "mlocker.h" #include "generic-ops.h" #include "hex.h" #include "span.h" @@ -66,7 +66,7 @@ namespace crypto { friend class crypto_ops; }; - using secret_key = tools::scrubbed<ec_scalar>; + using secret_key = epee::mlocked<tools::scrubbed<ec_scalar>>; POD_CLASS public_keyV { std::vector<public_key> keys; @@ -99,6 +99,7 @@ namespace crypto { #pragma pack(pop) void hash_to_scalar(const void *data, size_t length, ec_scalar &res); + void random32_unbiased(unsigned char *bytes); static_assert(sizeof(ec_point) == 32 && sizeof(ec_scalar) == 32 && sizeof(public_key) == 32 && sizeof(secret_key) == 32 && @@ -277,11 +278,11 @@ namespace crypto { epee::to_hex::formatted(o, epee::as_byte_span(v)); return o; } - const static crypto::public_key null_pkey = boost::value_initialized<crypto::public_key>(); - const static crypto::secret_key null_skey = boost::value_initialized<crypto::secret_key>(); + const extern crypto::public_key null_pkey; + const extern crypto::secret_key null_skey; } CRYPTO_MAKE_HASHABLE(public_key) -CRYPTO_MAKE_HASHABLE(secret_key) +CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key) CRYPTO_MAKE_HASHABLE(key_image) CRYPTO_MAKE_COMPARABLE(signature) diff --git a/src/crypto/crypto_ops_builder/ietf.txt b/src/crypto/crypto_ops_builder/ietf.txt index 0736f71ec..609f5e75a 100644 --- a/src/crypto/crypto_ops_builder/ietf.txt +++ b/src/crypto/crypto_ops_builder/ietf.txt @@ -1,1402 +1,4 @@ - +https://tools.ietf.org/id/draft-josefsson-eddsa-ed25519-02.txt -[Docs] [txt|pdf] [Tracker] [Email] [Diff1] [Diff2] [Nits] - -Versions: 00 01 02 - -Network Working Group S. Josefsson -Internet-Draft SJD AB -Intended status: Informational N. Moeller -Expires: August 26, 2015 - February 22, 2015 - - - EdDSA and Ed25519 - - draft-josefsson-eddsa-ed25519-02 - - -Abstract - - The elliptic curve signature scheme EdDSA and one instance of it - called Ed25519 is described. An example implementation and test - vectors are provided. - -Status of This Memo - - This Internet-Draft is submitted in full conformance with the - provisions of BCP 78 and BCP 79. - - Internet-Drafts are working documents of the Internet Engineering - Task Force (IETF). Note that other groups may also distribute - working documents as Internet-Drafts. The list of current Internet- - Drafts is at http://datatracker.ietf.org/drafts/current/. - - Internet-Drafts are draft documents valid for a maximum of six months - and may be updated, replaced, or obsoleted by other documents at any - time. It is inappropriate to use Internet-Drafts as reference - material or to cite them other than as "work in progress." - - This Internet-Draft will expire on August 26, 2015. - -Copyright Notice - - Copyright (c) 2015 IETF Trust and the persons identified as the - document authors. All rights reserved. - - This document is subject to BCP 78 and the IETF Trust's Legal - Provisions Relating to IETF Documents - (http://trustee.ietf.org/license-info) in effect on the date of - publication of this document. Please review these documents - carefully, as they describe your rights and restrictions with respect - to this document. Code Components extracted from this document must - include Simplified BSD License text as described in Section 4.e of - the Trust Legal Provisions and are provided without warranty as - described in the Simplified BSD License. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 1] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -Table of Contents - - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Notation . . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 3. Background . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 4. EdDSA . . . . . . . . . . . . . . . . . . . . . . . . . . . . 4 - 4.1. Encoding . . . . . . . . . . . . . . . . . . . . . . . . 4 - 4.2. Keys . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 4.3. Sign . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 4.4. Verify . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 5. Ed25519 . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 5.1. Modular arithmetic . . . . . . . . . . . . . . . . . . . 6 - 5.2. Encoding . . . . . . . . . . . . . . . . . . . . . . . . 6 - 5.3. Decoding . . . . . . . . . . . . . . . . . . . . . . . . 6 - 5.4. Point addition . . . . . . . . . . . . . . . . . . . . . 7 - 5.5. Key Generation . . . . . . . . . . . . . . . . . . . . . 8 - 5.6. Sign . . . . . . . . . . . . . . . . . . . . . . . . . . 8 - 5.7. Verify . . . . . . . . . . . . . . . . . . . . . . . . . 9 - 5.8. Python illustration . . . . . . . . . . . . . . . . . . . 9 - 6. Test Vectors for Ed25519 . . . . . . . . . . . . . . . . . . 14 - 7. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . 17 - 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 18 - 9. Security Considerations . . . . . . . . . . . . . . . . . . . 18 - 9.1. Side-channel leaks . . . . . . . . . . . . . . . . . . . 18 - 10. References . . . . . . . . . . . . . . . . . . . . . . . . . 18 - 10.1. Normative References . . . . . . . . . . . . . . . . . . 18 - 10.2. Informative References . . . . . . . . . . . . . . . . . 18 - Appendix A. Ed25519 Python Library . . . . . . . . . . . . . . . 19 - Appendix B. Library driver . . . . . . . . . . . . . . . . . . . 23 - Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 24 - -1. Introduction - - - The Edwards-curve Digital Signature Algorithm (EdDSA) is a variant of - Schnorr's signature system with Twisted Edwards curves. EdDSA needs - to be instantiated with certain parameters and this document describe - Ed25519 - an instantiation of EdDSA in a curve over GF(2^255-19). To - facilitate adoption in the Internet community of Ed25519, this - document describe the signature scheme in an implementation-oriented - way, and we provide sample code and test vectors. - - The advantages with EdDSA and Ed25519 include: - - 1. High-performance on a variety of platforms. - - 2. Does not require the use of a unique random number for each - signature. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 2] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 3. More resilient to side-channel attacks. - - 4. Small public keys (32 bytes) and signatures (64 bytes). - - 5. The formulas are "strongly unified", i.e., they are valid for all - points on the curve, with no exceptions. This obviates the need - for EdDSA to perform expensive point validation on untrusted - public values. - - 6. Collision resilience, meaning that hash-function collisions do - not break this system. - - For further background, see the original EdDSA paper [EDDSA]. - -2. Notation - - - The following notation is used throughout the document: - - GF(p) finite field with p elements - - x^y x multiplied by itself y times - - B generator of the group or subgroup of interest - - n B B added to itself n times. - - h_i the i'th bit of h - - a || b (bit-)string a concatenated with (bit-)string b - -3. Background - - - EdDSA is defined using an elliptic curve over GF(p) of the form - - -x^2 + y^2 = 1 + d x^2 y^2 - - In general, p could be a prime power, but it is usually chosen as a - prime number. It is required that p = 1 modulo 4 (which implies that - -1 is a square modulo p) and that d is a non-square modulo p. For - Ed25519, the curve used is equivalent to Curve25519 [CURVE25519], - under a change of coordinates, which means that the difficulty of the - discrete logarithm problem is the same as for Curve25519. - - Points on this curve form a group under addition, (x3, y3) = (x1, y1) - + (x2, y2), with the formulas - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 3] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - x1 y2 + x2 y1 y1 y2 + x1 x2 - x3 = -------------------, y3 = ------------------- - 1 + d x1 x2 y1 y2 1 - d x1 x2 y1 y2 - - The neutral element in the group is (0, 1). - - Unlike manyy other curves used for cryptographic applications, these - formulas are "strongly unified": they are valid for all points on the - curve, with no exceptions. In particular, the denominators are non- - zero for all input points. - - There are more efficient formulas, which are still strongly unified, - which use homogeneous coordinates to avoid the expensive modulo p - inversions. See [Faster-ECC] and [Edwards-revisited]. - -4. EdDSA - - - EdDSA is a digital signature system with several parameters. The - generic EdDSA digital signature system is normally not implemented - directly, but instead a particular instance of EdDSA (like Ed25519) - is implemented. A precise explanation of the generic EdDSA is thus - not particulary useful for implementers, but for background and - completeness, a succint description of the generic EdDSA algorithm is - given here. - - EdDSA has seven parameters: - - 1. an integer b >= 10. - - 2. a cryptographic hash function H producing 2b-bit outputs. - - 3. a prime power p congruent to 1 modulo 4. - - 4. a (b-1)-bit encoding of elements of the finite field GF(p). - - 5. a non-square element d of GF(p) - - 6. an element B != (0,1) of the set E = { (x,y) is a member of GF(p) - x GF(p) such that -x^2 + y^2 = 1 + dx^2y^2 }. - - 7. a prime q, of size b-3 bits, such that qB = (0, 1), i.e., q is - the order of B or a multiple thereof. - -4.1. Encoding - - - An element (x,y) of E is encoded as a b-bit string called ENC(x,y) - which is the (b-1)-bit encoding of y concatenated with one bit that - is 1 if x is negative and 0 if x is not negative. Negative elements - - - -Josefsson & Moeller Expires August 26, 2015 [Page 4] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - of GF(q) are those x which the (b-1)-bit encoding of x is - lexicographically larger than the (b-1)-bit encoding of -x. - -4.2. Keys - - - An EdDSA secret key is a b-bit string k. Let the hash H(k) = (h_0, - h_1, ..., h_(2b-1)) determine an integer a which is 2^(b-2) plus the - sum of m = 2^i * h_i for all i equal or larger than 3 and equal to or - less than b-3 such that m is a member of the set { 2^(b-2), 2^(b-2) + - 8, ..., 2^(b-1) - 8 }. The EdDSA public key is ENC(A) = ENC(aB). - The bits h_b, ..., h_(2b-1) is used below during signing. - -4.3. Sign - - - The signature of a message M under a secret key k is the 2b-bit - string ENC(R) || ENC'(S), where ENC'(S) is defined as the b-bit - little-endian encoding of S. R and S are derived as follows. First - define r = H(h_b, ... h_(2b-1)), M) interpreting 2b-bit strings in - little-endian form as integers in {0, 1, ..., 2^(2b)-1}. Let R=rB - and S=(r+H(ENC(R) || ENC(A) || M)a) mod l. - -4.4. Verify - - - To verify a signature ENC(R) || ENC'(S) on a message M under a public - key ENC(A), proceed as follows. Parse the inputs so that A and R is - an element of E, and S is a member of the set {0, 1, ..., l-1 }. - Compute H' = H(ENC(R) || ENC(A) || M) and check the group equation - 8SB = 8R + 8H'A in E. Verification is rejected if parsing fails or - the group equation does not hold. - -5. Ed25519 - - - Theoretically, Ed25519 is EdDSA instantiated with b=256, H being - SHA-512 [RFC4634], p is the prime 2^255-19, the 255-bit encoding of - GF(2^255-19) being the little-endian encoding of {0, 1, ..., - 2^255-20}, q is the prime 2^252 + 0x14def9dea2f79cd65812631a5cf5d3ed, - d = -121665/121666 which is a member of GF(p), and B is the unique - point (x, 4/5) in E for which x is "positive", which with the - encoding used simply means that the least significant bit of x is 0. - The curve p, prime q, d and B follows from [I-D.irtf-cfrg-curves]. - - Written out explicitly, B is the point (15112221349535400772501151409 - 588531511454012693041857206046113283949847762202, 4631683569492647816 - 9428394003475163141307993866256225615783033603165251855960). - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 5] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -5.1. Modular arithmetic - - - For advise on how to implement arithmetic modulo p = 2^255 - 1 - efficiently and securely, see Curve25519 [CURVE25519]. For inversion - modulo p, it is recommended to use the identity x^-1 = x^(p-2) (mod - p). - - For point decoding or "decompression", square roots modulo p are - needed. They can be computed using the Tonelli-Shanks algorithm, or - the special case for p = 5 (mod 8). To find a square root of a, - first compute the candidate root x = a^((p+3)/8) (mod p). Then there - are three cases: - - x^2 = a (mod p). Then x is a square root. - - x^2 = -a (mod p). Then 2^((p-1)/4) x is a square root. - - a is not a square modulo p. - -5.2. Encoding - - - All values are coded as octet strings, and integers are coded using - little endian convention. I.e., a 32-octet string h h[0],...h[31] - represents the integer h[0] + 2^8 h[1] + ... + 2^248 h[31]. - - A curve point (x,y), with coordiantes in the range 0 <= x,y < p, is - coded as follows. First encode the y-coordinate as a little-endian - string of 32 octets. The most significant bit of the final octet is - always zero. To form the encoding of the point, copy the least - significant bit of the x-coordinate to the most significant bit of - the final octet. - -5.3. Decoding - - - Decoding a point, given as a 32-octet string, is a little more - complicated. - - 1. First interpret the string as an integer in little-endian - representation. Bit 255 of this number is the least significant - bit of the x-coordinate, and denote this value x_0. The - y-coordinate is recovered simply by clearing this bit. If the - resulting value is >= p, decoding fails. - - 2. To recover the x coordinate, the curve equation implies x^2 = - (y^2 - 1) / (d y^2 + 1) (mod p). Since d is a non-square and -1 - is a square, the numerator, (d y^2 + 1), is always invertible - modulo p. Let u = y^2 - 1 and v = d y^2 + 1. To compute the - square root of (u/v), the first step is to compute the candidate - - - -Josefsson & Moeller Expires August 26, 2015 [Page 6] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - root x = (u/v)^((p+3)/8). This can be done using the following - trick, to use a single modular powering for both the inversion of - v and the square root: - - (p+3)/8 3 (p-5)/8 - x = (u/v) = u v (u v^7) (mod p) - - 3. Again, there are three cases: - - 1. If v x^2 = u (mod p), x is a square root. - - 2. If v x^2 = -u (mod p), set x <-- x 2^((p-1)/4), which is a - square root. - - 3. Otherwise, no square root exists modulo p, and decoding - fails. - - 4. Finally, use the x_0 bit to select the right square root. If x = - 0, and x_0 = 1, decoding fails. Otherwise, if x_0 != x mod 2, - set x <-- p - x. Return the decoded point (x,y). - -5.4. Point addition - - - For point addition, the following method is recommended. A point - (x,y) is represented in extended homogeneous coordinates (X, Y, Z, - T), with x = X/Z, y = Y/Z, x y = T/Z. - - The following formulas for adding two points, (x3,y3) = - (x1,y1)+(x2,y2) are described in [Edwards-revisited], section 3.1. - They are strongly unified, i.e., they work for any pair of valid - input points. - - A = (Y1-X1)*(Y2-X2) - B = (Y1+X1)*(Y2+X2) - C = T1*2*d*T2 - D = Z1*2*Z2 - E = B-A - F = D-C - G = D+C - H = B+A - X3 = E*F - Y3 = G*H - T3 = E*H - Z3 = F*G - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 7] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -5.5. Key Generation - - - The secret is 32 octets (256 bits, corresponding to b) of - cryptographically-secure random data. See [RFC4086] for a discussion - about randomness. - - The 32-byte public key is generated by the following steps. - - 1. Hash the 32-byte secret using SHA-512, storing the digest in a - 64-octet large buffer, denoted h. Only the lower 32 bytes are - used for generating the public key. - - 2. Prune the buffer. In C terminology: - - h[0] &= ~0x07; - h[31] &= 0x7F; - h[31] |= 0x40; - - 3. Interpret the buffer as the little-endian integer, forming a - secret scalar a. Perform a known-base-point scalar - multiplication a B. - - 4. The public key A is the encoding of the point aB. First encode - the y coordinate (in the range 0 <= y < p) as a little-endian - string of 32 octets. The most significant bit of the final octet - is always zero. To form the encoding of the point aB, copy the - least significant bit of the x coordinate to the most significant - bit of the final octet. The result is the public key. - -5.6. Sign - - - The imputs to the signing procedure is the secret key, a 32-octet - string, and a message M of arbitrary size. - - 1. Hash the secret key, 32-octets, using SHA-512. Let h denote the - resulting digest. Construct the secret scalar a from the first - half of the digest, and the corresponding public key A, as - described in the previous section. Let prefix denote the second - half of the hash digest, h[32],...,h[63]. - - 2. Compute SHA-512(prefix || M), where M is the message to be - signed. Interpret the 64-octet digest as a little-endian integer - r. - - 3. Compute the point rB. For efficiency, do this by first reducing - r modulo q, the group order of B. Let the string R be the - encoding of this point. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 8] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 4. Compute SHA512(R || A || M), and interpret the 64-octet digest as - a little-endian integer k. - - 5. Compute s = (r + k a) mod q. For efficiency, again reduce k - modulo q first. - - 6. Form the signature of the concatenation of R (32 octets) and the - little-endian encoding of s (32 octets, three most significant - bits of the final octets always zero). - -5.7. Verify - - - 1. To verify a signature on a message M, first split the signature - into two 32-octet halves. Decode the first half as a point R, - and the second half as an integer s, in the range 0 <= s < q. If - the decoding fails, the signature is invalid. - - 2. Compute SHA512(R || A || M), and interpret the 64-octet digest as - a little-endian integer k. - - 3. Check the group equation 8s B = 8 R + 8k A. It's sufficient, but - not required, to instead check s B = R + k A. - -5.8. Python illustration - - - The rest of this section describes how Ed25519 can be implemented in - Python (version 3.2 or later) for illustration. See appendix A for - the complete implementation and appendix B for a test-driver to run - it through some test vectors. - - First some preliminaries that will be needed. - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 9] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - import hashlib - - def sha512(s): - return hashlib.sha512(s).digest() - - # Base field Z_p - p = 2**255 - 19 - - def modp_inv(x): - return pow(x, p-2, p) - - # Curve constant - d = -121665 * modp_inv(121666) % p - - # Group order - q = 2**252 + 27742317777372353535851937790883648493 - - def sha512_modq(s): - return int.from_bytes(sha512(s), "little") % q - - Then follows functions to perform point operations. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 10] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -# Points are represented as tuples (X, Y, Z, T) of extended coordinates, -# with x = X/Z, y = Y/Z, x*y = T/Z - -def point_add(P, Q): - A = (P[1]-P[0])*(Q[1]-Q[0]) % p - B = (P[1]+P[0])*(Q[1]+Q[0]) % p - C = 2 * P[3] * Q[3] * d % p - D = 2 * P[2] * Q[2] % p - E = B-A - F = D-C - G = D+C - H = B+A - return (E*F, G*H, F*G, E*H) - -# Computes Q = s * Q -def point_mul(s, P): - Q = (0, 1, 1, 0) # Neutral element - while s > 0: - # Is there any bit-set predicate? - if s & 1: - Q = point_add(Q, P) - P = point_add(P, P) - s >>= 1 - return Q - -def point_equal(P, Q): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - if (P[0] * Q[2] - Q[0] * P[2]) % p != 0: - return False - if (P[1] * Q[2] - Q[1] * P[2]) % p != 0: - return False - return True - - Now follows functions for point compression. - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 11] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -# Square root of -1 -modp_sqrt_m1 = pow(2, (p-1) // 4, p) - -# Compute corresponding x coordinate, with low bit corresponding to sign, -# or return None on failure -def recover_x(y, sign): - x2 = (y*y-1) * modp_inv(d*y*y+1) - if x2 == 0: - if sign: - return None - else: - return 0 - - # Compute square root of x2 - x = pow(x2, (p+3) // 8, p) - if (x*x - x2) % p != 0: - x = x * modp_sqrt_m1 % p - if (x*x - x2) % p != 0: - return None - - if (x & 1) != sign: - x = p - x - return x - -# Base point -g_y = 4 * modp_inv(5) % p -g_x = recover_x(g_y, 0) -G = (g_x, g_y, 1, g_x * g_y % p) - -def point_compress(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - -def point_decompress(s): - if len(s) != 32: - raise Exception("Invalid input length for decompression") - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - - x = recover_x(y, sign) - if x is None: - return None - else: - return (x, y, 1, x*y % p) - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 12] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - These are functions for manipulating the secret. - - def secret_expand(secret): - if len(secret) != 32: - raise Exception("Bad size of private key") - h = sha512(secret) - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= (1 << 254) - return (a, h[32:]) - - def secret_to_public(secret): - (a, dummy) = secret_expand(secret) - return point_compress(point_mul(a, G)) - - The signature function works as below. - - def sign(secret, msg): - a, prefix = secret_expand(secret) - A = point_compress(point_mul(a, G)) - r = sha512_modq(prefix + msg) - R = point_mul(r, G) - Rs = point_compress(R) - h = sha512_modq(Rs + A + msg) - s = (r + h * a) % q - return Rs + int.to_bytes(s, 32, "little") - - And finally the verification function. - - def verify(public, msg, signature): - if len(public) != 32: - raise Exception("Bad public-key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - h = sha512_modq(Rs + public + msg) - sB = point_mul(s, G) - hA = point_mul(h, A) - return point_equal(sB, point_add(R, hA)) - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 13] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -6. Test Vectors for Ed25519 - - - Below is a sequence of octets with test vectors for the the Ed25519 - signature algorithm. The octets are hex encoded and whitespace is - inserted for readability. Private keys are 64 bytes, public keys 32 - bytes, message of arbitrary length, and signatures are 64 bytes. The - test vectors are taken from [ED25519-TEST-VECTORS] (but we removed - the public key as a suffix of the secret key, and removed the message - from the signature) and [ED25519-LIBGCRYPT-TEST-VECTORS]. - - -----TEST 1 - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - d75a980182b10ab7d54bfed3c964073a - 0ee172f3daa62325af021a68f707511a - - MESSAGE (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - - -----TEST 2 - SECRET KEY: - 4ccd089b28ff96da9db6c346ec114e0f - 5b8a319f35aba624da8cf6ed4fb8a6fb - - PUBLIC KEY: - 3d4017c3e843895a92b70aa74d1b7ebc - 9c982ccf2ec4968cc0cd55f12af4660c - - MESSAGE (length 1 byte): - 72 - - SIGNATURE: - 92a009a9f0d4cab8720e820b5f642540 - a2b27b5416503f8fb3762223ebdb69da - 085ac1e43e15996e458f3613d0f11d8c - 387b2eaeb4302aeeb00d291612bb0c00 - - -----TEST 3 - SECRET KEY: - c5aa8df43f9f837bedb7442f31dcb7b1 - - - -Josefsson & Moeller Expires August 26, 2015 [Page 14] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 66d38535076f094b85ce3a2e0b4458f7 - - PUBLIC KEY: - fc51cd8e6218a1a38da47ed00230f058 - 0816ed13ba3303ac5deb911548908025 - - MESSAGE (length 2 bytes): - af82 - - SIGNATURE: - 6291d657deec24024827e69c3abe01a3 - 0ce548a284743a445e3680d7db5ac3ac - 18ff9b538d16f290ae67f760984dc659 - 4a7c15e9716ed28dc027beceea1ec40a - - -----TEST 1024 - SECRET KEY: - f5e5767cf153319517630f226876b86c - 8160cc583bc013744c6bf255f5cc0ee5 - - PUBLIC KEY: - 278117fc144c72340f67d0f2316e8386 - ceffbf2b2428c9c51fef7c597f1d426e - - MESSAGE: - 08b8b2b733424243760fe426a4b54908 - 632110a66c2f6591eabd3345e3e4eb98 - fa6e264bf09efe12ee50f8f54e9f77b1 - e355f6c50544e23fb1433ddf73be84d8 - 79de7c0046dc4996d9e773f4bc9efe57 - 38829adb26c81b37c93a1b270b20329d - 658675fc6ea534e0810a4432826bf58c - 941efb65d57a338bbd2e26640f89ffbc - 1a858efcb8550ee3a5e1998bd177e93a - 7363c344fe6b199ee5d02e82d522c4fe - ba15452f80288a821a579116ec6dad2b - 3b310da903401aa62100ab5d1a36553e - 06203b33890cc9b832f79ef80560ccb9 - a39ce767967ed628c6ad573cb116dbef - efd75499da96bd68a8a97b928a8bbc10 - 3b6621fcde2beca1231d206be6cd9ec7 - aff6f6c94fcd7204ed3455c68c83f4a4 - 1da4af2b74ef5c53f1d8ac70bdcb7ed1 - 85ce81bd84359d44254d95629e9855a9 - 4a7c1958d1f8ada5d0532ed8a5aa3fb2 - d17ba70eb6248e594e1a2297acbbb39d - 502f1a8c6eb6f1ce22b3de1a1f40cc24 - 554119a831a9aad6079cad88425de6bd - - - -Josefsson & Moeller Expires August 26, 2015 [Page 15] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - e1a9187ebb6092cf67bf2b13fd65f270 - 88d78b7e883c8759d2c4f5c65adb7553 - 878ad575f9fad878e80a0c9ba63bcbcc - 2732e69485bbc9c90bfbd62481d9089b - eccf80cfe2df16a2cf65bd92dd597b07 - 07e0917af48bbb75fed413d238f5555a - 7a569d80c3414a8d0859dc65a46128ba - b27af87a71314f318c782b23ebfe808b - 82b0ce26401d2e22f04d83d1255dc51a - ddd3b75a2b1ae0784504df543af8969b - e3ea7082ff7fc9888c144da2af58429e - c96031dbcad3dad9af0dcbaaaf268cb8 - fcffead94f3c7ca495e056a9b47acdb7 - 51fb73e666c6c655ade8297297d07ad1 - ba5e43f1bca32301651339e22904cc8c - 42f58c30c04aafdb038dda0847dd988d - cda6f3bfd15c4b4c4525004aa06eeff8 - ca61783aacec57fb3d1f92b0fe2fd1a8 - 5f6724517b65e614ad6808d6f6ee34df - f7310fdc82aebfd904b01e1dc54b2927 - 094b2db68d6f903b68401adebf5a7e08 - d78ff4ef5d63653a65040cf9bfd4aca7 - 984a74d37145986780fc0b16ac451649 - de6188a7dbdf191f64b5fc5e2ab47b57 - f7f7276cd419c17a3ca8e1b939ae49e4 - 88acba6b965610b5480109c8b17b80e1 - b7b750dfc7598d5d5011fd2dcc5600a3 - 2ef5b52a1ecc820e308aa342721aac09 - 43bf6686b64b2579376504ccc493d97e - 6aed3fb0f9cd71a43dd497f01f17c0e2 - cb3797aa2a2f256656168e6c496afc5f - b93246f6b1116398a346f1a641f3b041 - e989f7914f90cc2c7fff357876e506b5 - 0d334ba77c225bc307ba537152f3f161 - 0e4eafe595f6d9d90d11faa933a15ef1 - 369546868a7f3a45a96768d40fd9d034 - 12c091c6315cf4fde7cb68606937380d - b2eaaa707b4c4185c32eddcdd306705e - 4dc1ffc872eeee475a64dfac86aba41c - 0618983f8741c5ef68d3a101e8a3b8ca - c60c905c15fc910840b94c00a0b9d0 - - SIGNATURE: - 0aab4c900501b3e24d7cdf4663326a3a - 87df5e4843b2cbdb67cbf6e460fec350 - aa5371b1508f9f4528ecea23c436d94b - 5e8fcd4f681e30a6ac00a9704a188a03 - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 16] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - -----TEST 1A - -----An additional test with the data from test 1 but using an - -----uncompressed public key. - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - 0455d0e09a2b9d34292297e08d60d0f6 - 20c513d47253187c24b12786bd777645 - ce1a5107f7681a02af2523a6daf372e1 - 0e3a0764c9d3fe4bd5b70ab18201985a - d7 - - MSG (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - - -----TEST 1B - -----An additional test with the data from test 1 but using an - -----compressed prefix. - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - 40d75a980182b10ab7d54bfed3c96407 - 3a0ee172f3daa62325af021a68f70751 - 1a - - MESSAGE (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - ----- - -7. Acknowledgements - - - Feedback on this document was received from Werner Koch and Damien - Miller. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 17] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -8. IANA Considerations - - - None. - -9. Security Considerations - - -9.1. Side-channel leaks - - - For implementations performing signatures, secrecy of the key is - fundamental. It is possible to protect against some side-channel - attacks by ensuring that the implementation executes exactly the same - sequence of instructions and performs exactly the same memory - accesses, for any value of the secret key. - - To make an implementation side-channel silent in this way, the modulo - p arithmetic must not use any data-dependent branches, e.g., related - to carry propagation. Side channel-silent point addition is - straight-forward, thanks to the unified formulas. - - Scalar multiplication, multiplying a point by an integer, needs some - additional effort to implement in a side-channel silent manner. One - simple approach is to implement a side-channel silent conditional - assignment, and use together with the binary algorithm to examine one - bit of the integer at a time. - - Note that the example implementation in this document does not - attempt to be side-channel silent. - -10. References - - -10.1. Normative References - - - [RFC4634] Eastlake, D. and T. Hansen, "US Secure Hash Algorithms - (SHA and HMAC-SHA)", RFC 4634, July 2006. - - [I-D.irtf-cfrg-curves] - Langley, A., Salz, R., and S. Turner, "Elliptic Curves for - Security", draft-irtf-cfrg-curves-01 (work in progress), - January 2015. - -10.2. Informative References - - - [RFC4086] Eastlake, D., Schiller, J., and S. Crocker, "Randomness - Requirements for Security", BCP 106, RFC 4086, June 2005. - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 18] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - [EDDSA] Bernstein, D., Duif, N., Lange, T., Schwabe, P., and B. - Yang, "High-speed high-security signatures", WWW - http://ed25519.cr.yp.to/ed25519-20110926.pdf, September - 2011. - - [Faster-ECC] - Bernstein, D. and T. Lange, "Faster addition and doubling - on elliptic curves", WWW http://eprint.iacr.org/2007/286, - July 2007. - - [Edwards-revisited] - Hisil, H., Wong, K., Carter, G., and E. Dawson, "Twisted - Edwards Curves Revisited", WWW - http://eprint.iacr.org/2008/522, December 2008. - - [CURVE25519] - Bernstein, D., "Curve25519: new Diffie-Hellman speed - records", WWW http://cr.yp.to/ecdh.html, February 2006. - - [ED25519-TEST-VECTORS] - Bernstein, D., Duif, N., Lange, T., Schwabe, P., and B. - Yang, "Ed25519 test vectors", WWW - http://ed25519.cr.yp.to/python/sign.input, July 2011. - - [ED25519-LIBGCRYPT-TEST-VECTORS] - Koch, W., "Ed25519 Libgcrypt test vectors", WWW - http://git.gnupg.org/cgi- - bin/gitweb.cgi?p=libgcrypt.git;a=blob;f=tests/t-ed25519.in - p;h=e13566f826321eece65e02c593bc7d885b3dbe23;hb=refs/ - heads/master, July 2014. - -Appendix A. Ed25519 Python Library - - - Below is an example implementation of Ed25519 written in Python, - version 3.2 or higher is required. - -# Loosely based on the public domain code at -# http://ed25519.cr.yp.to/software.html -# -# Needs python-3.2 - -import hashlib - - -def sha512(s): - return hashlib.sha512(s).digest() - -# Base field Z_p - - - -Josefsson & Moeller Expires August 26, 2015 [Page 19] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -p = 2**255 - 19 - - -def modp_inv(x): - return pow(x, p-2, p) - -# Curve constant -d = -121665 * modp_inv(121666) % p - -# Group order -q = 2**252 + 27742317777372353535851937790883648493 - - -def sha512_modq(s): - return int.from_bytes(sha512(s), "little") % q - -# Points are represented as tuples (X, Y, Z, T) of extended coordinates, -# with x = X/Z, y = Y/Z, x*y = T/Z - - -def point_add(P, Q): - A = (P[1]-P[0])*(Q[1]-Q[0]) % p - B = (P[1]+P[0])*(Q[1]+Q[0]) % p - C = 2 * P[3] * Q[3] * d % p - D = 2 * P[2] * Q[2] % p - E = B-A - F = D-C - G = D+C - H = B+A - return (E*F, G*H, F*G, E*H) - - -# Computes Q = s * Q -def point_mul(s, P): - Q = (0, 1, 1, 0) # Neutral element - while s > 0: - # Is there any bit-set predicate? - if s & 1: - Q = point_add(Q, P) - P = point_add(P, P) - s >>= 1 - return Q - - -def point_equal(P, Q): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - if (P[0] * Q[2] - Q[0] * P[2]) % p != 0: - return False - - - -Josefsson & Moeller Expires August 26, 2015 [Page 20] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - if (P[1] * Q[2] - Q[1] * P[2]) % p != 0: - return False - return True - -# Square root of -1 -modp_sqrt_m1 = pow(2, (p-1) // 4, p) - - -# Compute corresponding x coordinate, with low bit corresponding to sign, -# or return None on failure -def recover_x(y, sign): - x2 = (y*y-1) * modp_inv(d*y*y+1) - if x2 == 0: - if sign: - return None - else: - return 0 - - # Compute square root of x2 - x = pow(x2, (p+3) // 8, p) - if (x*x - x2) % p != 0: - x = x * modp_sqrt_m1 % p - if (x*x - x2) % p != 0: - return None - - if (x & 1) != sign: - x = p - x - return x - -# Base point -g_y = 4 * modp_inv(5) % p -g_x = recover_x(g_y, 0) -G = (g_x, g_y, 1, g_x * g_y % p) - - -def point_compress(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - - -def point_decompress(s): - if len(s) != 32: - raise Exception("Invalid input length for decompression") - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - - - -Josefsson & Moeller Expires August 26, 2015 [Page 21] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - x = recover_x(y, sign) - if x is None: - return None - else: - return (x, y, 1, x*y % p) - - -def secret_expand(secret): - if len(secret) != 32: - raise Exception("Bad size of private key") - h = sha512(secret) - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= (1 << 254) - return (a, h[32:]) - - -def secret_to_public(secret): - (a, dummy) = secret_expand(secret) - return point_compress(point_mul(a, G)) - - -def sign(secret, msg): - a, prefix = secret_expand(secret) - A = point_compress(point_mul(a, G)) - r = sha512_modq(prefix + msg) - R = point_mul(r, G) - Rs = point_compress(R) - h = sha512_modq(Rs + A + msg) - s = (r + h * a) % q - return Rs + int.to_bytes(s, 32, "little") - - -def verify(public, msg, signature): - if len(public) != 32: - raise Exception("Bad public-key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - h = sha512_modq(Rs + public + msg) - sB = point_mul(s, G) - - - -Josefsson & Moeller Expires August 26, 2015 [Page 22] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - hA = point_mul(h, A) - return point_equal(sB, point_add(R, hA)) - -Appendix B. Library driver - - - Below is a command-line tool that uses the library above to perform - computations, for interactive use or for self-checking. - - import sys - import binascii - - from ed25519 import * - - def point_valid(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - assert (x*y - P[3]*zinv) % p == 0 - return (-x*x + y*y - 1 - d*x*x*y*y) % p == 0 - - assert point_valid(G) - Z = (0, 1, 1, 0) - assert point_valid(Z) - - assert point_equal(Z, point_add(Z, Z)) - assert point_equal(G, point_add(Z, G)) - assert point_equal(Z, point_mul(0, G)) - assert point_equal(G, point_mul(1, G)) - assert point_equal(point_add(G, G), point_mul(2, G)) - for i in range(0, 100): - assert point_valid(point_mul(i, G)) - assert point_equal(Z, point_mul(q, G)) - - def munge_string(s, pos, change): - return (s[:pos] + - int.to_bytes(s[pos] ^ change, 1, "little") + - s[pos+1:]) - - # Read a file in the format of - # http://ed25519.cr.yp.to/python/sign.input - lineno = 0 - while True: - line = sys.stdin.readline() - if not line: - break - lineno = lineno + 1 - print(lineno) - fields = line.split(":") - - - -Josefsson & Moeller Expires August 26, 2015 [Page 23] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - secret = (binascii.unhexlify(fields[0]))[:32] - public = binascii.unhexlify(fields[1]) - msg = binascii.unhexlify(fields[2]) - signature = binascii.unhexlify(fields[3])[:64] - - assert public == secret_to_public(secret) - assert signature == sign(secret, msg) - assert verify(public, msg, signature) - if len(msg) == 0: - bad_msg = b"x" - else: - bad_msg = munge_string(msg, len(msg) // 3, 4) - assert not verify(public, bad_msg, signature) - bad_signature = munge_string(signature, 20, 8) - assert not verify(public, msg, bad_signature) - bad_signature = munge_string(signature, 40, 16) - assert not verify(public, msg, bad_signature) - -Authors' Addresses - - Simon Josefsson - SJD AB - - Email: simon@josefsson.org - URI: http://josefsson.org/ - - - Niels Moeller - - Email: nisse@lysator.liu.se - - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 24] - - - -Html markup produced by rfcmarkup 1.113, available from https://tools.ietf.org/tools/rfcmarkup/ +Note: This draft is now superseded by https://datatracker.ietf.org/doc/rfc8032/ +(review of the differences is left as an exercise for the reader) diff --git a/src/crypto/crypto_ops_builder/ref10/README.md b/src/crypto/crypto_ops_builder/ref10/README.md new file mode 100644 index 000000000..59193305b --- /dev/null +++ b/src/crypto/crypto_ops_builder/ref10/README.md @@ -0,0 +1,4 @@ +This code comes from Daniel J. Bernstein's SUPERCOP source, +released in the public domain. + +[http://ed25519.cr.yp.to/software.html](http://ed25519.cr.yp.to/software.html) diff --git a/src/crypto/crypto_ops_builder/ref10/description b/src/crypto/crypto_ops_builder/ref10/description index cbfcb2cba..99f747747 100644 --- a/src/crypto/crypto_ops_builder/ref10/description +++ b/src/crypto/crypto_ops_builder/ref10/description @@ -1,2 +1,2 @@ EdDSA signatures using Curve25519 -from http://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 +from https://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py index 9b55d260d..0ed97d5f4 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py @@ -1,5 +1,5 @@ #assumes you have gnu sed, osx sed might need slight syntax changeo -#c.f. http://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files +#c.f. https://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files #written by shen-noether monero research labs @@ -8,7 +8,7 @@ import glob #for copy files import textwrap #for comments etc print("make sure you have cat and grep installed") -print("also assumes gnu sed syntax, c.f. :http://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files") +print("also assumes gnu sed syntax, c.f. :https://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files") print("I believe osx may have slightly different version of sed") print("maybe someone smart can replace the sed with perl..") diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/description b/src/crypto/crypto_ops_builder/ref10CommentedCombined/description index fadc9f9af..9407b400a 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/description +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/description @@ -2,6 +2,6 @@ shen_ed25519_ref10 MakeCryptoOps.py makes crypto-ops.c in the Monero source from the ref10 implementation EdDSA signatures using Curve25519 -from http://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 +from https://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 Commented / combined by Shen Noether, Monero Research Lab diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 62bc758c9..42b98706e 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -33,19 +33,30 @@ #include <cstddef> #include <cstring> #include <functional> +#include <sodium/crypto_verify_32.h> #define CRYPTO_MAKE_COMPARABLE(type) \ namespace crypto { \ inline bool operator==(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) == 0; \ + return !memcmp(&_v1, &_v2, sizeof(_v1)); \ } \ inline bool operator!=(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) != 0; \ + return !operator==(_v1, _v2); \ } \ } -#define CRYPTO_MAKE_HASHABLE(type) \ -CRYPTO_MAKE_COMPARABLE(type) \ +#define CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +namespace crypto { \ + inline bool operator==(const type &_v1, const type &_v2) { \ + static_assert(sizeof(_v1) == 32, "constant time comparison is only implenmted for 32 bytes"); \ + return crypto_verify_32((const unsigned char*)&_v1, (const unsigned char*)&_v2) == 0; \ + } \ + inline bool operator!=(const type &_v1, const type &_v2) { \ + return !operator==(_v1, _v2); \ + } \ +} + +#define CRYPTO_DEFINE_HASH_FUNCTIONS(type) \ namespace crypto { \ static_assert(sizeof(std::size_t) <= sizeof(type), "Size of " #type " must be at least that of size_t"); \ inline std::size_t hash_value(const type &_v) { \ @@ -60,3 +71,12 @@ namespace std { \ } \ }; \ } + +#define CRYPTO_MAKE_HASHABLE(type) \ +CRYPTO_MAKE_COMPARABLE(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + +#define CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(type) \ +CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + diff --git a/src/crypto/groestl.c b/src/crypto/groestl.c index c8258add3..d5e2989a8 100644 --- a/src/crypto/groestl.c +++ b/src/crypto/groestl.c @@ -20,9 +20,15 @@ const uint8_t shift_Values[2][8] = {{0,1,2,3,4,5,6,7},{1,3,5,7,0,2,4,6}}; const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; +#if BYTE_ORDER == LITTLE_ENDIAN #define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1<<(8*amount_bytes))|(v2>>(8*(4-amount_bytes))); \ v2 = (v2<<(8*amount_bytes))|(v1>>(8*(4-amount_bytes))); \ v1 = temp_var;} +#else +#define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1>>(8*amount_bytes))|(v2<<(8*(4-amount_bytes))); \ + v2 = (v2>>(8*amount_bytes))|(v1<<(8*(4-amount_bytes))); \ + v1 = temp_var;} +#endif #define COLUMN(x,y,i,c0,c1,c2,c3,c4,c5,c6,c7,tv1,tv2,tu,tl,t) \ @@ -68,14 +74,14 @@ const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; static void RND512P(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; - x32[ 0] ^= 0x00000000^r; - x32[ 2] ^= 0x00000010^r; - x32[ 4] ^= 0x00000020^r; - x32[ 6] ^= 0x00000030^r; - x32[ 8] ^= 0x00000040^r; - x32[10] ^= 0x00000050^r; - x32[12] ^= 0x00000060^r; - x32[14] ^= 0x00000070^r; + x32[ 0] ^= SWAP32LE(0x00000000)^r; + x32[ 2] ^= SWAP32LE(0x00000010)^r; + x32[ 4] ^= SWAP32LE(0x00000020)^r; + x32[ 6] ^= SWAP32LE(0x00000030)^r; + x32[ 8] ^= SWAP32LE(0x00000040)^r; + x32[10] ^= SWAP32LE(0x00000050)^r; + x32[12] ^= SWAP32LE(0x00000060)^r; + x32[14] ^= SWAP32LE(0x00000070)^r; COLUMN(x,y, 0, 0, 2, 4, 6, 9, 11, 13, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 2, 4, 6, 8, 11, 13, 15, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 4, 6, 8, 10, 13, 15, 1, 3, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -91,21 +97,22 @@ static void RND512Q(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; x32[ 0] = ~x32[ 0]; - x32[ 1] ^= 0xffffffff^r; + x32[ 1] ^= SWAP32LE(0xffffffff)^r; x32[ 2] = ~x32[ 2]; - x32[ 3] ^= 0xefffffff^r; + x32[ 3] ^= SWAP32LE(0xefffffff)^r; x32[ 4] = ~x32[ 4]; - x32[ 5] ^= 0xdfffffff^r; + x32[ 5] ^= SWAP32LE(0xdfffffff)^r; x32[ 6] = ~x32[ 6]; - x32[ 7] ^= 0xcfffffff^r; + x32[ 7] ^= SWAP32LE(0xcfffffff)^r; x32[ 8] = ~x32[ 8]; - x32[ 9] ^= 0xbfffffff^r; + x32[ 9] ^= SWAP32LE(0xbfffffff)^r; x32[10] = ~x32[10]; - x32[11] ^= 0xafffffff^r; + x32[11] ^= SWAP32LE(0xafffffff)^r; x32[12] = ~x32[12]; - x32[13] ^= 0x9fffffff^r; + x32[13] ^= SWAP32LE(0x9fffffff)^r; x32[14] = ~x32[14]; - x32[15] ^= 0x8fffffff^r; + x32[15] ^= SWAP32LE(0x8fffffff)^r; + COLUMN(x,y, 0, 2, 6, 10, 14, 1, 5, 9, 13, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 4, 8, 12, 0, 3, 7, 11, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 6, 10, 14, 2, 5, 9, 13, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -130,28 +137,28 @@ static void F512(uint32_t *h, const uint32_t *m) { } /* compute Q(m) */ - RND512Q((uint8_t*)z, y, 0x00000000); - RND512Q((uint8_t*)y, z, 0x01000000); - RND512Q((uint8_t*)z, y, 0x02000000); - RND512Q((uint8_t*)y, z, 0x03000000); - RND512Q((uint8_t*)z, y, 0x04000000); - RND512Q((uint8_t*)y, z, 0x05000000); - RND512Q((uint8_t*)z, y, 0x06000000); - RND512Q((uint8_t*)y, z, 0x07000000); - RND512Q((uint8_t*)z, y, 0x08000000); - RND512Q((uint8_t*)y, Qtmp, 0x09000000); + RND512Q((uint8_t*)z, y, SWAP32LE(0x00000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x01000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x02000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x03000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x04000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x05000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x06000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x07000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x08000000)); + RND512Q((uint8_t*)y, Qtmp, SWAP32LE(0x09000000)); /* compute P(h+m) */ - RND512P((uint8_t*)Ptmp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, Ptmp, 0x00000009); + RND512P((uint8_t*)Ptmp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, Ptmp, SWAP32LE(0x00000009)); /* compute P(h+m) + Q(m) + h */ for (i = 0; i < 2*COLS512; i++) { @@ -188,16 +195,16 @@ static void OutputTransformation(hashState *ctx) { for (j = 0; j < 2*COLS512; j++) { temp[j] = ctx->chaining[j]; } - RND512P((uint8_t*)temp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, temp, 0x00000009); + RND512P((uint8_t*)temp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, temp, SWAP32LE(0x00000009)); for (j = 0; j < 2*COLS512; j++) { ctx->chaining[j] ^= temp[j]; } @@ -213,7 +220,7 @@ static void Init(hashState* ctx) { } /* set initial value */ - ctx->chaining[2*COLS512-1] = u32BIG((uint32_t)HASH_BIT_LEN); + ctx->chaining[2*COLS512-1] = SWAP32LE(u32BIG((uint32_t)HASH_BIT_LEN)); /* set other variables */ ctx->buf_ptr = 0; diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index c4b368584..53594c569 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -29,7 +29,10 @@ #ifndef __tables_h #define __tables_h +#include "common/int-util.h" + +#if BYTE_ORDER == LITTLE_ENDIAN const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05eee, 0xee99c7b0, 0x8d8c7af6, 0xf68df78c, 0xd17e8ff, 0xff0de517, 0xbddc0ad6, 0xd6bdb7dc, 0xb1c816de, 0xdeb1a7c8, 0x54fc6d91, 0x915439fc , 0x50f09060, 0x6050c0f0, 0x3050702, 0x2030405, 0xa9e02ece, 0xcea987e0, 0x7d87d156, 0x567dac87, 0x192bcce7, 0xe719d52b, 0x62a613b5, 0xb56271a6, 0xe6317c4d, 0x4de69a31, 0x9ab559ec, 0xec9ac3b5 , 0x45cf408f, 0x8f4505cf, 0x9dbca31f, 0x1f9d3ebc, 0x40c04989, 0x894009c0, 0x879268fa, 0xfa87ef92, 0x153fd0ef, 0xef15c53f, 0xeb2694b2, 0xb2eb7f26, 0xc940ce8e, 0x8ec90740, 0xb1de6fb, 0xfb0bed1d @@ -62,5 +65,39 @@ const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05 , 0xb6c1ec2d, 0x2db65ac1, 0x22665a3c, 0x3c227866, 0x92adb815, 0x15922aad, 0x2060a9c9, 0xc9208960, 0x49db5c87, 0x874915db, 0xff1ab0aa, 0xaaff4f1a, 0x7888d850, 0x5078a088, 0x7a8e2ba5, 0xa57a518e , 0x8f8a8903, 0x38f068a, 0xf8134a59, 0x59f8b213, 0x809b9209, 0x980129b, 0x1739231a, 0x1a173439, 0xda751065, 0x65daca75, 0x315384d7, 0xd731b553, 0xc651d584, 0x84c61351, 0xb8d303d0, 0xd0b8bbd3 , 0xc35edc82, 0x82c31f5e, 0xb0cbe229, 0x29b052cb, 0x7799c35a, 0x5a77b499, 0x11332d1e, 0x1e113c33, 0xcb463d7b, 0x7bcbf646, 0xfc1fb7a8, 0xa8fc4b1f, 0xd6610c6d, 0x6dd6da61, 0x3a4e622c, 0x2c3a584e}; +#else +const uint32_t T[512] = {0xc632f4a5, 0xf497a5c6, 0xf86f9784, 0x97eb84f8, 0xee5eb099, 0xb0c799ee, 0xf67a8c8d, 0x8cf78df6, 0xffe8170d, 0x17e50dff, 0xd60adcbd, 0xdcb7bdd6, 0xde16c8b1, 0xc8a7b1de, 0x916dfc54, 0xfc395491 +, 0x6090f050, 0xf0c05060, 0x02070503, 0x05040302, 0xce2ee0a9, 0xe087a9ce, 0x56d1877d, 0x87ac7d56, 0xe7cc2b19, 0x2bd519e7, 0xb513a662, 0xa67162b5, 0x4d7c31e6, 0x319ae64d, 0xec59b59a, 0xb5c39aec +, 0x8f40cf45, 0xcf05458f, 0x1fa3bc9d, 0xbc3e9d1f, 0x8949c040, 0xc0094089, 0xfa689287, 0x92ef87fa, 0xefd03f15, 0x3fc515ef, 0xb29426eb, 0x267febb2, 0x8ece40c9, 0x4007c98e, 0xfbe61d0b, 0x1ded0bfb +, 0x416e2fec, 0x2f82ec41, 0xb31aa967, 0xa97d67b3, 0x5f431cfd, 0x1cbefd5f, 0x456025ea, 0x258aea45, 0x23f9dabf, 0xda46bf23, 0x535102f7, 0x02a6f753, 0xe445a196, 0xa1d396e4, 0x9b76ed5b, 0xed2d5b9b +, 0x75285dc2, 0x5deac275, 0xe1c5241c, 0x24d91ce1, 0x3dd4e9ae, 0xe97aae3d, 0x4cf2be6a, 0xbe986a4c, 0x6c82ee5a, 0xeed85a6c, 0x7ebdc341, 0xc3fc417e, 0xf5f30602, 0x06f102f5, 0x8352d14f, 0xd11d4f83 +, 0x688ce45c, 0xe4d05c68, 0x515607f4, 0x07a2f451, 0xd18d5c34, 0x5cb934d1, 0xf9e11808, 0x18e908f9, 0xe24cae93, 0xaedf93e2, 0xab3e9573, 0x954d73ab, 0x6297f553, 0xf5c45362, 0x2a6b413f, 0x41543f2a +, 0x081c140c, 0x14100c08, 0x9563f652, 0xf6315295, 0x46e9af65, 0xaf8c6546, 0x9d7fe25e, 0xe2215e9d, 0x30487828, 0x78602830, 0x37cff8a1, 0xf86ea137, 0x0a1b110f, 0x11140f0a, 0x2febc4b5, 0xc45eb52f +, 0x0e151b09, 0x1b1c090e, 0x247e5a36, 0x5a483624, 0x1badb69b, 0xb6369b1b, 0xdf98473d, 0x47a53ddf, 0xcda76a26, 0x6a8126cd, 0x4ef5bb69, 0xbb9c694e, 0x7f334ccd, 0x4cfecd7f, 0xea50ba9f, 0xbacf9fea +, 0x123f2d1b, 0x2d241b12, 0x1da4b99e, 0xb93a9e1d, 0x58c49c74, 0x9cb07458, 0x3446722e, 0x72682e34, 0x3641772d, 0x776c2d36, 0xdc11cdb2, 0xcda3b2dc, 0xb49d29ee, 0x2973eeb4, 0x5b4d16fb, 0x16b6fb5b +, 0xa4a501f6, 0x0153f6a4, 0x76a1d74d, 0xd7ec4d76, 0xb714a361, 0xa37561b7, 0x7d3449ce, 0x49face7d, 0x52df8d7b, 0x8da47b52, 0xdd9f423e, 0x42a13edd, 0x5ecd9371, 0x93bc715e, 0x13b1a297, 0xa2269713 +, 0xa6a204f5, 0x0457f5a6, 0xb901b868, 0xb86968b9, 0x00000000, 0x00000000, 0xc1b5742c, 0x74992cc1, 0x40e0a060, 0xa0806040, 0xe3c2211f, 0x21dd1fe3, 0x793a43c8, 0x43f2c879, 0xb69a2ced, 0x2c77edb6 +, 0xd40dd9be, 0xd9b3bed4, 0x8d47ca46, 0xca01468d, 0x671770d9, 0x70ced967, 0x72afdd4b, 0xdde44b72, 0x94ed79de, 0x7933de94, 0x98ff67d4, 0x672bd498, 0xb09323e8, 0x237be8b0, 0x855bde4a, 0xde114a85 +, 0xbb06bd6b, 0xbd6d6bbb, 0xc5bb7e2a, 0x7e912ac5, 0x4f7b34e5, 0x349ee54f, 0xedd73a16, 0x3ac116ed, 0x86d254c5, 0x5417c586, 0x9af862d7, 0x622fd79a, 0x6699ff55, 0xffcc5566, 0x11b6a794, 0xa7229411 +, 0x8ac04acf, 0x4a0fcf8a, 0xe9d93010, 0x30c910e9, 0x040e0a06, 0x0a080604, 0xfe669881, 0x98e781fe, 0xa0ab0bf0, 0x0b5bf0a0, 0x78b4cc44, 0xccf04478, 0x25f0d5ba, 0xd54aba25, 0x4b753ee3, 0x3e96e34b +, 0xa2ac0ef3, 0x0e5ff3a2, 0x5d4419fe, 0x19bafe5d, 0x80db5bc0, 0x5b1bc080, 0x0580858a, 0x850a8a05, 0x3fd3ecad, 0xec7ead3f, 0x21fedfbc, 0xdf42bc21, 0x70a8d848, 0xd8e04870, 0xf1fd0c04, 0x0cf904f1 +, 0x63197adf, 0x7ac6df63, 0x772f58c1, 0x58eec177, 0xaf309f75, 0x9f4575af, 0x42e7a563, 0xa5846342, 0x20705030, 0x50403020, 0xe5cb2e1a, 0x2ed11ae5, 0xfdef120e, 0x12e10efd, 0xbf08b76d, 0xb7656dbf +, 0x8155d44c, 0xd4194c81, 0x18243c14, 0x3c301418, 0x26795f35, 0x5f4c3526, 0xc3b2712f, 0x719d2fc3, 0xbe8638e1, 0x3867e1be, 0x35c8fda2, 0xfd6aa235, 0x88c74fcc, 0x4f0bcc88, 0x2e654b39, 0x4b5c392e +, 0x936af957, 0xf93d5793, 0x55580df2, 0x0daaf255, 0xfc619d82, 0x9de382fc, 0x7ab3c947, 0xc9f4477a, 0xc827efac, 0xef8bacc8, 0xba8832e7, 0x326fe7ba, 0x324f7d2b, 0x7d642b32, 0xe642a495, 0xa4d795e6 +, 0xc03bfba0, 0xfb9ba0c0, 0x19aab398, 0xb3329819, 0x9ef668d1, 0x6827d19e, 0xa322817f, 0x815d7fa3, 0x44eeaa66, 0xaa886644, 0x54d6827e, 0x82a87e54, 0x3bdde6ab, 0xe676ab3b, 0x0b959e83, 0x9e16830b +, 0x8cc945ca, 0x4503ca8c, 0xc7bc7b29, 0x7b9529c7, 0x6b056ed3, 0x6ed6d36b, 0x286c443c, 0x44503c28, 0xa72c8b79, 0x8b5579a7, 0xbc813de2, 0x3d63e2bc, 0x1631271d, 0x272c1d16, 0xad379a76, 0x9a4176ad +, 0xdb964d3b, 0x4dad3bdb, 0x649efa56, 0xfac85664, 0x74a6d24e, 0xd2e84e74, 0x1436221e, 0x22281e14, 0x92e476db, 0x763fdb92, 0x0c121e0a, 0x1e180a0c, 0x48fcb46c, 0xb4906c48, 0xb88f37e4, 0x376be4b8 +, 0x9f78e75d, 0xe7255d9f, 0xbd0fb26e, 0xb2616ebd, 0x43692aef, 0x2a86ef43, 0xc435f1a6, 0xf193a6c4, 0x39dae3a8, 0xe372a839, 0x31c6f7a4, 0xf762a431, 0xd38a5937, 0x59bd37d3, 0xf274868b, 0x86ff8bf2 +, 0xd5835632, 0x56b132d5, 0x8b4ec543, 0xc50d438b, 0x6e85eb59, 0xebdc596e, 0xda18c2b7, 0xc2afb7da, 0x018e8f8c, 0x8f028c01, 0xb11dac64, 0xac7964b1, 0x9cf16dd2, 0x6d23d29c, 0x49723be0, 0x3b92e049 +, 0xd81fc7b4, 0xc7abb4d8, 0xacb915fa, 0x1543faac, 0xf3fa0907, 0x09fd07f3, 0xcfa06f25, 0x6f8525cf, 0xca20eaaf, 0xea8fafca, 0xf47d898e, 0x89f38ef4, 0x476720e9, 0x208ee947, 0x10382818, 0x28201810 +, 0x6f0b64d5, 0x64ded56f, 0xf0738388, 0x83fb88f0, 0x4afbb16f, 0xb1946f4a, 0x5cca9672, 0x96b8725c, 0x38546c24, 0x6c702438, 0x575f08f1, 0x08aef157, 0x732152c7, 0x52e6c773, 0x9764f351, 0xf3355197 +, 0xcbae6523, 0x658d23cb, 0xa125847c, 0x84597ca1, 0xe857bf9c, 0xbfcb9ce8, 0x3e5d6321, 0x637c213e, 0x96ea7cdd, 0x7c37dd96, 0x611e7fdc, 0x7fc2dc61, 0x0d9c9186, 0x911a860d, 0x0f9b9485, 0x941e850f +, 0xe04bab90, 0xabdb90e0, 0x7cbac642, 0xc6f8427c, 0x712657c4, 0x57e2c471, 0xcc29e5aa, 0xe583aacc, 0x90e373d8, 0x733bd890, 0x06090f05, 0x0f0c0506, 0xf7f40301, 0x03f501f7, 0x1c2a3612, 0x3638121c +, 0xc23cfea3, 0xfe9fa3c2, 0x6a8be15f, 0xe1d45f6a, 0xaebe10f9, 0x1047f9ae, 0x69026bd0, 0x6bd2d069, 0x17bfa891, 0xa82e9117, 0x9971e858, 0xe8295899, 0x3a536927, 0x6974273a, 0x27f7d0b9, 0xd04eb927 +, 0xd9914838, 0x48a938d9, 0xebde3513, 0x35cd13eb, 0x2be5ceb3, 0xce56b32b, 0x22775533, 0x55443322, 0xd204d6bb, 0xd6bfbbd2, 0xa9399070, 0x904970a9, 0x07878089, 0x800e8907, 0x33c1f2a7, 0xf266a733 +, 0x2decc1b6, 0xc15ab62d, 0x3c5a6622, 0x6678223c, 0x15b8ad92, 0xad2a9215, 0xc9a96020, 0x608920c9, 0x875cdb49, 0xdb154987, 0xaab01aff, 0x1a4fffaa, 0x50d88878, 0x88a07850, 0xa52b8e7a, 0x8e517aa5 +, 0x03898a8f, 0x8a068f03, 0x594a13f8, 0x13b2f859, 0x09929b80, 0x9b128009, 0x1a233917, 0x3934171a, 0x651075da, 0x75cada65, 0xd7845331, 0x53b531d7, 0x84d551c6, 0x5113c684, 0xd003d3b8, 0xd3bbb8d0 +, 0x82dc5ec3, 0x5e1fc382, 0x29e2cbb0, 0xcb52b029, 0x5ac39977, 0x99b4775a, 0x1e2d3311, 0x333c111e, 0x7b3d46cb, 0x46f6cb7b, 0xa8b71ffc, 0x1f4bfca8, 0x6d0c61d6, 0x61dad66d, 0x2c624e3a, 0x4e583a2c}; +#endif #endif /* __tables_h */ diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index afbace726..107988d2b 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -43,8 +43,8 @@ #elif defined(_MSC_VER) #include <assert.h> #include <stdlib.h> -// http://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc -// http://msdn.microsoft.com/en-us/library/bb918180.aspx +// https://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc +// https://msdn.microsoft.com/en-us/library/bb918180.aspx #pragma section(".CRT$XCT", read) #define INITIALIZER(name) \ static void __cdecl name(void); \ diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index de8e2a5b3..b095b5ce2 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -5,6 +5,7 @@ #include <stdio.h> #include <stdlib.h> #include <unistd.h> +#include "common/int-util.h" #include "hash-ops.h" #include "keccak.h" @@ -105,7 +106,7 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) for ( ; inlen >= rsiz; inlen -= rsiz, in += rsiz) { for (i = 0; i < rsizw; i++) - st[i] ^= ((uint64_t *) in)[i]; + st[i] ^= swap64le(((uint64_t *) in)[i]); keccakf(st, KECCAK_ROUNDS); } @@ -121,14 +122,93 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) temp[rsiz - 1] |= 0x80; for (i = 0; i < rsizw; i++) - st[i] ^= ((uint64_t *) temp)[i]; + st[i] ^= swap64le(((uint64_t *) temp)[i]); keccakf(st, KECCAK_ROUNDS); - memcpy(md, st, mdlen); + if (((size_t)mdlen % sizeof(uint64_t)) != 0) + { + local_abort("Bad keccak use"); + } + memcpy_swap64le(md, st, mdlen/sizeof(uint64_t)); } void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md) { keccak(in, inlen, md, sizeof(state_t)); } + +#define KECCAK_FINALIZED 0x80000000 +#define KECCAK_BLOCKLEN 136 +#define KECCAK_WORDS 17 +#define KECCAK_DIGESTSIZE 32 +#define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0))) +#define KECCAK_PROCESS_BLOCK(st, block) { \ + for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \ + ((st))[i_] ^= swap64le(((block))[i_]); \ + }; \ + keccakf(st, KECCAK_ROUNDS); } + + +void keccak_init(KECCAK_CTX * ctx){ + memset(ctx, 0, sizeof(KECCAK_CTX)); +} + +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen){ + if (ctx->rest & KECCAK_FINALIZED) { + local_abort("Bad keccak use"); + } + + const size_t idx = ctx->rest; + ctx->rest = (ctx->rest + inlen) % KECCAK_BLOCKLEN; + + // fill partial block + if (idx) { + size_t left = KECCAK_BLOCKLEN - idx; + memcpy((char*)ctx->message + idx, in, (inlen < left ? inlen : left)); + if (inlen < left) return; + + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + + in += left; + inlen -= left; + } + + const bool is_aligned = IS_ALIGNED_64(in); + while (inlen >= KECCAK_BLOCKLEN) { + const uint64_t* aligned_message_block; + if (is_aligned) { + aligned_message_block = (uint64_t*)in; + } else { + memcpy(ctx->message, in, KECCAK_BLOCKLEN); + aligned_message_block = ctx->message; + } + + KECCAK_PROCESS_BLOCK(ctx->hash, aligned_message_block); + in += KECCAK_BLOCKLEN; + inlen -= KECCAK_BLOCKLEN; + } + if (inlen) { + memcpy(ctx->message, in, inlen); + } +} + +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){ + if (!(ctx->rest & KECCAK_FINALIZED)) + { + // clear the rest of the data queue + memset((char*)ctx->message + ctx->rest, 0, KECCAK_BLOCKLEN - ctx->rest); + ((char*)ctx->message)[ctx->rest] |= 0x01; + ((char*)ctx->message)[KECCAK_BLOCKLEN - 1] |= 0x80; + + // process final block + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + ctx->rest = KECCAK_FINALIZED; // mark context as finalized + } + + static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, ""); + static_assert(KECCAK_DIGESTSIZE % sizeof(uint64_t) == 0, ""); + if (md) { + memcpy_swap64le(md, ctx->hash, KECCAK_DIGESTSIZE / sizeof(uint64_t)); + } +} diff --git a/src/crypto/keccak.h b/src/crypto/keccak.h index fb9d8bd04..9123c7a3b 100644 --- a/src/crypto/keccak.h +++ b/src/crypto/keccak.h @@ -15,6 +15,17 @@ #define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) #endif +// SHA3 Algorithm context. +typedef struct KECCAK_CTX +{ + // 1600 bits algorithm hashing state + uint64_t hash[25]; + // 1088-bit buffer for leftovers, block size = 136 B for 256-bit keccak + uint64_t message[17]; + // count of bytes in the message[] buffer + size_t rest; +} KECCAK_CTX; + // compute a keccak hash (md) of given byte length from "in" void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen); @@ -23,4 +34,7 @@ void keccakf(uint64_t st[25], int norounds); void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md); +void keccak_init(KECCAK_CTX * ctx); +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen); +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md); #endif diff --git a/src/crypto/oaes_lib.c b/src/crypto/oaes_lib.c index 9e31ebf46..210f5d43a 100644 --- a/src/crypto/oaes_lib.c +++ b/src/crypto/oaes_lib.c @@ -33,14 +33,15 @@ #include <stdlib.h> #include <stdio.h> -// OS X, FreeBSD, and OpenBSD don't need malloc.h +// OS X, FreeBSD, OpenBSD and NetBSD don't need malloc.h #if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) \ - && !defined(__DragonFly__) + && !defined(__DragonFly__) && !defined(__NetBSD__) #include <malloc.h> #endif -// ANDROID, FreeBSD, and OpenBSD also don't need timeb.h -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) +// ANDROID, FreeBSD, OpenBSD and NetBSD also don't need timeb.h +#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) \ + && !defined(__NetBSD__) #include <sys/timeb.h> #else #include <sys/time.h> @@ -473,7 +474,7 @@ OAES_RET oaes_sprintf( #ifdef OAES_HAVE_ISAAC static void oaes_get_seed( char buf[RANDSIZ + 1] ) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; @@ -505,7 +506,7 @@ static void oaes_get_seed( char buf[RANDSIZ + 1] ) #else static uint32_t oaes_get_seed(void) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 35e98f2f5..40cfb0461 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -38,6 +38,7 @@ #include "common/int-util.h" #include "hash-ops.h" #include "oaes_lib.h" +#include "variant2_int_sqrt.h" #define MEMORY (1 << 21) // 2MB scratchpad #define ITER (1 << 20) @@ -50,7 +51,7 @@ extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expa extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); #define VARIANT1_1(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ const uint8_t tmp = ((const uint8_t*)(p))[11]; \ static const uint32_t table = 0x75310; \ @@ -59,7 +60,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_2(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ xor64(p, tweak1_2); \ } while(0) @@ -67,7 +68,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_CHECK() \ do if (length < 43) \ { \ - fprintf(stderr, "Cryptonight variants need at least 43 bytes of data"); \ + fprintf(stderr, "Cryptonight variant 1 needs at least 43 bytes of data"); \ _exit(1); \ } while(0) @@ -75,7 +76,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_PORTABLE_INIT() \ uint8_t tweak1_2[8]; \ - do if (variant > 0) \ + do if (variant == 1) \ { \ VARIANT1_CHECK(); \ memcpy(&tweak1_2, &state.hs.b[192], sizeof(tweak1_2)); \ @@ -83,11 +84,135 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_INIT64() \ - if (variant > 0) \ + if (variant == 1) \ { \ VARIANT1_CHECK(); \ } \ - const uint64_t tweak1_2 = variant > 0 ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + const uint64_t tweak1_2 = (variant == 1) ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + +#define VARIANT2_INIT64() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \ + U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_PORTABLE_INIT() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + memcpy(b + AES_BLOCK_SIZE, state.hs.b + 64, AES_BLOCK_SIZE); \ + xor64(b + AES_BLOCK_SIZE, state.hs.b + 80); \ + xor64(b + AES_BLOCK_SIZE + 8, state.hs.b + 88); \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_SSE2(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const __m128i chunk1 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10))); \ + const __m128i chunk2 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20))); \ + const __m128i chunk3 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30))); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10)), _mm_add_epi64(chunk3, _b1)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20)), _mm_add_epi64(chunk1, _b)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30)), _mm_add_epi64(chunk2, _a)); \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_NEON(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const uint64x2_t chunk1 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x10))); \ + const uint64x2_t chunk2 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x20))); \ + const uint64x2_t chunk3 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x30))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x10)), vaddq_u64(chunk3, vreinterpretq_u64_u8(_b1))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x20)), vaddq_u64(chunk1, vreinterpretq_u64_u8(_b))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x30)), vaddq_u64(chunk2, vreinterpretq_u64_u8(_a))); \ + } while (0) + +#define VARIANT2_PORTABLE_SHUFFLE_ADD(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + uint64_t* chunk1 = U64((base_ptr) + ((offset) ^ 0x10)); \ + uint64_t* chunk2 = U64((base_ptr) + ((offset) ^ 0x20)); \ + uint64_t* chunk3 = U64((base_ptr) + ((offset) ^ 0x30)); \ + \ + const uint64_t chunk1_old[2] = { chunk1[0], chunk1[1] }; \ + \ + uint64_t b1[2]; \ + memcpy(b1, b + 16, 16); \ + chunk1[0] = chunk3[0] + b1[0]; \ + chunk1[1] = chunk3[1] + b1[1]; \ + \ + uint64_t a0[2]; \ + memcpy(a0, a, 16); \ + chunk3[0] = chunk2[0] + a0[0]; \ + chunk3[1] = chunk2[1] + a0[1]; \ + \ + uint64_t b0[2]; \ + memcpy(b0, b, 16); \ + chunk2[0] = chunk1_old[0] + b0[0]; \ + chunk2[1] = chunk1_old[1] + b0[1]; \ + } while (0) + +#define VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr) \ + ((uint64_t*)(b))[0] ^= division_result ^ (sqrt_result << 32); \ + { \ + const uint64_t dividend = ((uint64_t*)(ptr))[1]; \ + const uint32_t divisor = (((uint64_t*)(ptr))[0] + (uint32_t)(sqrt_result << 1)) | 0x80000001UL; \ + division_result = ((uint32_t)(dividend / divisor)) + \ + (((uint64_t)(dividend % divisor)) << 32); \ + } \ + const uint64_t sqrt_input = ((uint64_t*)(ptr))[0] + division_result + +#define VARIANT2_INTEGER_MATH_SSE2(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while(0) + +#if defined DBL_MANT_DIG && (DBL_MANT_DIG >= 50) + // double precision floating point type has enough bits of precision on current platform + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_FP64(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while (0) +#else + // double precision floating point type is not good enough on current platform + // fall back to the reference code (integer only) + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_REF(); \ + } while (0) +#endif + +#define VARIANT2_2_PORTABLE() \ + if (variant >= 2) { \ + xor_blocks(long_state + (j ^ 0x10), d); \ + xor_blocks(d, long_state + (j ^ 0x20)); \ + } + +#define VARIANT2_2() \ + do if (variant >= 2) \ + { \ + *U64(hp_state + (j ^ 0x10)) ^= hi; \ + *(U64(hp_state + (j ^ 0x10)) + 1) ^= lo; \ + hi ^= *U64(hp_state + (j ^ 0x20)); \ + lo ^= *(U64(hp_state + (j ^ 0x20)) + 1); \ + } while (0) + #if !defined NO_AES && (defined(__x86_64__) || (defined(_MSC_VER) && defined(_WIN64))) // Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI @@ -164,19 +289,23 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp * This code is based upon an optimized implementation by dga. */ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _b = _mm_xor_si128(_b, _c); \ - _mm_store_si128(R128(&hp_state[j]), _b); \ + _mm_store_si128(R128(&hp_state[j]), _mm_xor_si128(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_INTEGER_MATH_SSE2(b, c); \ __mul(); \ + VARIANT2_2(); \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ #if defined(_MSC_VER) @@ -309,7 +438,7 @@ STATIC INLINE void aes_256_assist2(__m128i* t1, __m128i * t3) * CPU AES support. * For more information about these functions, see page 19 of Intel's AES instructions * white paper: - * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/aes-instructions-set-white-paper.pdf + * https://www.intel.com/content/dam/doc/white-paper/advanced-encryption-standard-new-instructions-set-paper.pdf * * @param key the input 128 bit key * @param expandedKey An output buffer to hold the generated key schedule @@ -492,7 +621,7 @@ void slow_hash_allocate_state(void) MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); #else #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ - defined(__DragonFly__) + defined(__DragonFly__) || defined(__NetBSD__) hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, 0, 0); #else @@ -558,7 +687,7 @@ void slow_hash_free_state(void) * AES support on x86 CPUs. * * A diagram of the inner loop of this function can be found at - * http://www.cs.cmu.edu/~dga/crypto/xmr/cryptonight.png + * https://www.cs.cmu.edu/~dga/crypto/xmr/cryptonight.png * * @param data the data to hash * @param length the length in bytes of the data @@ -570,10 +699,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - __m128i _a, _b, _c; + __m128i _a, _b, _b1, _c; uint64_t hi, lo; size_t i, j; @@ -599,6 +728,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -637,6 +767,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = _mm_load_si128(R128(b)); + _b1 = _mm_load_si128(R128(b) + 1); // Two independent versions, one with AES, one without, to ensure that // the useAes test is only performed once, not every iteration. if(useAes) @@ -761,19 +892,23 @@ union cn_slow_hash_state _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - _b = veorq_u8(_b, _c); \ - vst1q_u8(&hp_state[j], _b); \ + vst1q_u8(&hp_state[j], veorq_u8(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ __mul(); \ + VARIANT2_2(); \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ @@ -905,17 +1040,44 @@ STATIC INLINE void aes_pseudo_round_xor(const uint8_t *in, uint8_t *out, const u } } +#ifdef FORCE_USE_HEAP +STATIC INLINE void* aligned_malloc(size_t size, size_t align) +{ + void *result; +#ifdef _MSC_VER + result = _aligned_malloc(size, align); +#else + if (posix_memalign(&result, align, size)) result = NULL; +#endif + return result; +} + +STATIC INLINE void aligned_free(void *ptr) +{ +#ifdef _MSC_VER + _aligned_free(ptr); +#else + free(ptr); +#endif +} +#endif /* FORCE_USE_HEAP */ + void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) { RDATA_ALIGN16 uint8_t expandedKey[240]; + +#ifndef FORCE_USE_HEAP RDATA_ALIGN16 uint8_t hp_state[MEMORY]; +#else + uint8_t *hp_state = (uint8_t *)aligned_malloc(MEMORY,16); +#endif uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - uint8x16_t _a, _b, _c, zero = {0}; + uint8x16_t _a, _b, _b1, _c, zero = {0}; uint64_t hi, lo; size_t i, j; @@ -936,6 +1098,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -959,7 +1122,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = vld1q_u8((const uint8_t *)b); - + _b1 = vld1q_u8(((const uint8_t *)b) + AES_BLOCK_SIZE); for(i = 0; i < ITER / 2; i++) { @@ -993,6 +1156,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(state.init, text, INIT_SIZE_BYTE); hash_permutation(&state.hs); extra_hashes[state.hs.b[0] & 3](&state, 200, hash); + +#ifdef FORCE_USE_HEAP + aligned_free(hp_state); +#endif } #else /* aarch64 && crypto */ @@ -1075,6 +1242,11 @@ __asm__ __volatile__( #endif /* !aarch64 */ #endif // NO_OPTIMIZED_MULTIPLY_ON_ARM +STATIC INLINE void copy_block(uint8_t* dst, const uint8_t* src) +{ + memcpy(dst, src, AES_BLOCK_SIZE); +} + STATIC INLINE void sum_half_blocks(uint8_t* a, const uint8_t* b) { uint64_t a0, a1, b0, b1; @@ -1109,7 +1281,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c[AES_BLOCK_SIZE]; + uint8_t c1[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; uint8_t aes_key[AES_KEY_SIZE]; RDATA_ALIGN16 uint8_t expandedKey[256]; @@ -1127,8 +1301,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #ifndef FORCE_USE_HEAP uint8_t long_state[MEMORY]; #else - uint8_t *long_state = NULL; - long_state = (uint8_t *)malloc(MEMORY); + uint8_t *long_state = (uint8_t *)malloc(MEMORY); #endif if (prehashed) { @@ -1138,11 +1311,12 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } memcpy(text, state.init, INIT_SIZE_BYTE); - VARIANT1_INIT64(); - aes_ctx = (oaes_ctx *) oaes_alloc(); oaes_key_import_data(aes_ctx, state.hs.b, AES_KEY_SIZE); + VARIANT1_INIT64(); + VARIANT2_INIT64(); + // use aligned data memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len); for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) @@ -1163,23 +1337,34 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #define state_index(x) ((*(uint32_t *) x) & MASK) // Iteration 1 - p = &long_state[state_index(a)]; + j = state_index(a); + p = &long_state[j]; aesb_single_round(p, p, a); + copy_block(c1, p); - xor_blocks(b, p); - swap_blocks(b, p); - swap_blocks(a, b); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + xor_blocks(p, b); VARIANT1_1(p); // Iteration 2 - p = &long_state[state_index(a)]; - - mul(a, p, d); - sum_half_blocks(b, d); - swap_blocks(b, p); - xor_blocks(b, p); - swap_blocks(a, b); - VARIANT1_2(U64(p) + 1); + j = state_index(c1); + p = &long_state[j]; + copy_block(c, p); + + VARIANT2_PORTABLE_INTEGER_MATH(c, c1); + mul(c1, c, d); + VARIANT2_2_PORTABLE(); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + sum_half_blocks(a, d); + swap_blocks(a, c); + xor_blocks(a, c); + VARIANT1_2(U64(c) + 1); + copy_block(p, c); + + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1294,12 +1479,18 @@ union cn_slow_hash_state { #pragma pack(pop) void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed) { +#ifndef FORCE_USE_HEAP uint8_t long_state[MEMORY]; +#else + uint8_t *long_state = (uint8_t *)malloc(MEMORY); +#endif + union cn_slow_hash_state state; uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; - uint8_t c[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c1[AES_BLOCK_SIZE]; + uint8_t c2[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; size_t i, j; uint8_t aes_key[AES_KEY_SIZE]; @@ -1315,6 +1506,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int aes_ctx = (oaes_ctx *) oaes_alloc(); VARIANT1_PORTABLE_INIT(); + VARIANT2_PORTABLE_INIT(); oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { @@ -1324,9 +1516,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } - for (i = 0; i < 16; i++) { - a[i] = state.k[ i] ^ state.k[32 + i]; - b[i] = state.k[16 + i] ^ state.k[48 + i]; + for (i = 0; i < AES_BLOCK_SIZE; i++) { + a[i] = state.k[ i] ^ state.k[AES_BLOCK_SIZE * 2 + i]; + b[i] = state.k[AES_BLOCK_SIZE + i] ^ state.k[AES_BLOCK_SIZE * 3 + i]; } for (i = 0; i < ITER / 2; i++) { @@ -1335,26 +1527,33 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int * next address <-+ */ /* Iteration 1 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - aesb_single_round(c, c, a); - xor_blocks(b, c); - swap_blocks(b, c); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); - VARIANT1_1(&long_state[j * AES_BLOCK_SIZE]); + j = e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c1, &long_state[j]); + aesb_single_round(c1, c1, a); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + copy_block(&long_state[j], c1); + xor_blocks(&long_state[j], b); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + VARIANT1_1(&long_state[j]); /* Iteration 2 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - mul(a, c, d); - sum_half_blocks(b, d); - swap_blocks(b, c); - xor_blocks(b, c); - VARIANT1_2(c + 8); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); + j = e2i(c1, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c2, &long_state[j]); + VARIANT2_PORTABLE_INTEGER_MATH(c2, c1); + mul(c1, c2, d); + VARIANT2_2_PORTABLE(); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + swap_blocks(a, c1); + sum_half_blocks(c1, d); + swap_blocks(c1, c2); + xor_blocks(c1, c2); + VARIANT1_2(c2 + 8); + copy_block(&long_state[j], c2); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, a); + copy_block(a, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1370,6 +1569,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int /*memcpy(hash, &state, 32);*/ extra_hashes[state.hs.b[0] & 3](&state, 200, hash); oaes_free((OAES_CTX **) &aes_ctx); + +#ifdef FORCE_USE_HEAP + free(long_state); +#endif } #endif diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index e6d6a267c..b2dc3ffb2 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -36,7 +36,8 @@ #ifdef _MSC_VER #include <malloc.h> -#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) +#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) \ + && !defined(__NetBSD__) #include <alloca.h> #else #include <stdlib.h> @@ -67,7 +68,7 @@ size_t tree_hash_cnt(size_t count) { } void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { -// The blockchain block at height 202612 http://monerochain.info/block/bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698 +// The blockchain block at height 202612 https://moneroblocks.info/block/202612 // contained 514 transactions, that triggered bad calculation of variable "cnt" in the original version of this function // as from CryptoNote code. // diff --git a/src/crypto/variant2_int_sqrt.h b/src/crypto/variant2_int_sqrt.h new file mode 100644 index 000000000..b405bb798 --- /dev/null +++ b/src/crypto/variant2_int_sqrt.h @@ -0,0 +1,163 @@ +#ifndef VARIANT2_INT_SQRT_H +#define VARIANT2_INT_SQRT_H + +#include <math.h> +#include <float.h> + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2() \ + do { \ + const __m128i exp_double_bias = _mm_set_epi64x(0, 1023ULL << 52); \ + __m128d x = _mm_castsi128_pd(_mm_add_epi64(_mm_cvtsi64_si128(sqrt_input >> 12), exp_double_bias)); \ + x = _mm_sqrt_sd(_mm_setzero_pd(), x); \ + sqrt_result = (uint64_t)(_mm_cvtsi128_si64(_mm_sub_epi64(_mm_castpd_si128(x), exp_double_bias))) >> 19; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_FP64() \ + do { \ + sqrt_result = sqrt(sqrt_input + 18446744073709551616.0) * 2.0 - 8589934592.0; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_REF() \ + sqrt_result = integer_square_root_v2(sqrt_input) + +// Reference implementation of the integer square root for Cryptonight variant 2 +// Computes integer part of "sqrt(2^64 + n) * 2 - 2^33" +// +// In other words, given 64-bit unsigned integer n: +// 1) Write it as x = 1.NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN000... in binary (1 <= x < 2, all 64 bits of n are used) +// 2) Calculate sqrt(x) = 1.0RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR... (1 <= sqrt(x) < sqrt(2), so it will always start with "1.0" in binary) +// 3) Take 32 bits that come after "1.0" and return them as a 32-bit unsigned integer, discard all remaining bits +// +// Some sample inputs and outputs: +// +// Input | Output | Exact value of "sqrt(2^64 + n) * 2 - 2^33" +// -----------------|------------|------------------------------------------- +// 0 | 0 | 0 +// 2^32 | 0 | 0.99999999994179233909330885695244... +// 2^32 + 1 | 1 | 1.0000000001746229827200734316305... +// 2^50 | 262140 | 262140.00012206565608606978175873... +// 2^55 + 20963331 | 8384515 | 8384515.9999999997673963974959744... +// 2^55 + 20963332 | 8384516 | 8384516 +// 2^62 + 26599786 | 1013904242 | 1013904242.9999999999479374853545... +// 2^62 + 26599787 | 1013904243 | 1013904243.0000000001561875439364... +// 2^64 - 1 | 3558067407 | 3558067407.9041987696409179931096... + +// The reference implementation as it is now uses only unsigned int64 arithmetic, so it can't have undefined behavior +// It was tested once for all edge cases and confirmed correct +static inline uint32_t integer_square_root_v2(uint64_t n) +{ + uint64_t r = 1ULL << 63; + + for (uint64_t bit = 1ULL << 60; bit; bit >>= 2) + { + const bool b = (n < r + bit); + const uint64_t n_next = n - (r + bit); + const uint64_t r_next = r + bit * 2; + n = b ? n : n_next; + r = b ? r : r_next; + r >>= 1; + } + + return r * 2 + ((n > r) ? 1 : 0); +} + +/* +VARIANT2_INTEGER_MATH_SQRT_FIXUP checks that "r" is an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33" and adds or subtracts 1 if needed +It's hard to understand how it works, so here is a full calculation of formulas used in VARIANT2_INTEGER_MATH_SQRT_FIXUP + +The following inequalities must hold for r if it's an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33": +1) r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +2) r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 + +We need to check them using only unsigned integer arithmetic to avoid rounding errors and undefined behavior + +First inequality: r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 2^33 <= sqrt(2^64 + sqrt_input) * 2 +r/2 + 2^32 <= sqrt(2^64 + sqrt_input) +(r/2 + 2^32)^2 <= 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b)/2 + 2^32)^2 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 + 2^64 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 <= sqrt_input +(s*2+b)^2/4 + 2^32*r <= sqrt_input +(s^2*4+2*s*2*b+b^2)/4 + 2^32*r <= sqrt_input +s^2+s*b+b^2/4 + 2^32*r <= sqrt_input +s*(s+b) + b^2/4 + 2^32*r <= sqrt_input + +Let r2 = s*(s+b) + r*2^32 +r2 + b^2/4 <= sqrt_input + +If this inequality doesn't hold, then we must decrement r: IF "r2 + b^2/4 > sqrt_input" THEN r = r - 1 + +b can be 0 or 1 +If b is 0 then we need to compare "r2 > sqrt_input" +If b is 1 then b^2/4 = 0.25, so we need to compare "r2 + 0.25 > sqrt_input" +Since both r2 and sqrt_input are integers, we can safely replace it with "r2 + 1 > sqrt_input" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + b > sqrt_input" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + b", so it's safe to compare with sqrt_input: +r2 + b = s*(s+b) + r*2^32 + b +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067407*2^32 + 1 = 18446744068217447385 < 2^64 + +Second inequality: r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 1 + 2^33 > sqrt(2^64 + sqrt_input) * 2 +((r+1)/2 + 2^32)^2 > 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b+1)/2 + 2^32)^2 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + 2*(s*2+b+1)/2*2^32 + 2^64 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + (s*2+b+1)*2^32 > sqrt_input +(s*2+b+1)^2/4 + (r+1)*2^32 > sqrt_input +(s*2+(b+1))^2/4 + r*2^32 + 2^32 > sqrt_input +(s^2*4+2*s*2*(b+1)+(b+1)^2)/4 + r*2^32 + 2^32 > sqrt_input +s^2+s*(b+1)+(b+1)^2/4 + r*2^32 + 2^32 > sqrt_input +s*(s+b) + s + (b+1)^2/4 + r*2^32 + 2^32 > sqrt_input + +Let r2 = s*(s+b) + r*2^32 + +r2 + s + (b+1)^2/4 + 2^32 > sqrt_input +r2 + 2^32 + (b+1)^2/4 > sqrt_input - s + +If this inequality doesn't hold, then we must decrement r: IF "r2 + 2^32 + (b+1)^2/4 <= sqrt_input - s" THEN r = r - 1 +b can be 0 or 1 +If b is 0 then we need to compare "r2 + 2^32 + 1/4 <= sqrt_input - s" which is equal to "r2 + 2^32 < sqrt_input - s" because all numbers here are integers +If b is 1 then (b+1)^2/4 = 1, so we need to compare "r2 + 2^32 + 1 <= sqrt_input - s" which is also equal to "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + 2^32": +r2 + 2^32 = s*(s+b) + r*2^32 + 2^32 = s*(s+b) + (r+1)*2^32 +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067408*2^32 = 18446744072512414680 < 2^64 + +There will be no integer overflow when calculating "sqrt_input - s", i.e. "sqrt_input >= s" at all times: +s = trunc(r/2) = trunc(sqrt(2^64 + sqrt_input) - 2^32) < sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input > sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input + 2^32 - 1 > sqrt(2^64 + sqrt_input) +(sqrt_input + 2^32 - 1)^2 > sqrt_input + 2^64 +sqrt_input^2 + 2*sqrt_input*(2^32 - 1) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 2) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + (2^32-1)^2 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + 2^64-2^33+1 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) - 2^33 + 1 > 0 +This inequality is true if sqrt_input > 1 and it's easy to check that s = 0 if sqrt_input is 0 or 1, so there will be no integer overflow +*/ + +#define VARIANT2_INTEGER_MATH_SQRT_FIXUP(r) \ + do { \ + const uint64_t s = r >> 1; \ + const uint64_t b = r & 1; \ + const uint64_t r2 = (uint64_t)(s) * (s + b) + (r << 32); \ + r += ((r2 + b > sqrt_input) ? -1 : 0) + ((r2 + (1ULL << 32) < sqrt_input - s) ? 1 : 0); \ + } while(0) + +#endif diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index d50a9df67..21445959d 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -27,9 +27,13 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. if(APPLE) - find_library(IOKIT_LIBRARY IOKit) - mark_as_advanced(IOKIT_LIBRARY) - list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + if(DEPENDS) + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework ApplicationServices -framework AppKit -framework IOKit") + else() + find_library(IOKIT_LIBRARY IOKit) + mark_as_advanced(IOKIT_LIBRARY) + list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + endif() endif() set(cryptonote_basic_sources diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index bab991d19..edbc2c561 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -44,6 +44,9 @@ extern "C" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "account" +#define KEYS_ENCRYPTION_SALT 'k' + + using namespace std; DISABLE_VS_WARNINGS(4244 4345) @@ -60,7 +63,70 @@ DISABLE_VS_WARNINGS(4244 4345) m_device = &hwdev; MCDEBUG("device", "account_keys::set_device device type: "<<typeid(hwdev).name()); } + //----------------------------------------------------------------- + static void derive_key(const crypto::chacha_key &base_key, crypto::chacha_key &key) + { + static_assert(sizeof(base_key) == sizeof(crypto::hash), "chacha key and hash should be the same size"); + epee::mlocked<tools::scrubbed_arr<char, sizeof(base_key)+1>> data; + memcpy(data.data(), &base_key, sizeof(base_key)); + data[sizeof(base_key)] = KEYS_ENCRYPTION_SALT; + crypto::generate_chacha_key(data.data(), sizeof(data), key, 1); + } + //----------------------------------------------------------------- + static epee::wipeable_string get_key_stream(const crypto::chacha_key &base_key, const crypto::chacha_iv &iv, size_t bytes) + { + // derive a new key + crypto::chacha_key key; + derive_key(base_key, key); + // chacha + epee::wipeable_string buffer0(std::string(bytes, '\0')); + epee::wipeable_string buffer1 = buffer0; + crypto::chacha20(buffer0.data(), buffer0.size(), key, iv, buffer1.data()); + return buffer1; + } + //----------------------------------------------------------------- + void account_keys::xor_with_key_stream(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * (2 + m_multisig_keys.size())); + const char *ptr = key_stream.data(); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_spend_secret_key.data[i] ^= *ptr++; + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + for (crypto::secret_key &k: m_multisig_keys) + { + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + k.data[i] ^= *ptr++; + } + } + //----------------------------------------------------------------- + void account_keys::encrypt(const crypto::chacha_key &key) + { + m_encryption_iv = crypto::rand<crypto::chacha_iv>(); + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::decrypt(const crypto::chacha_key &key) + { + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::encrypt_viewkey(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * 2); + const char *ptr = key_stream.data(); + ptr += sizeof(crypto::secret_key); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + } + //----------------------------------------------------------------- + void account_keys::decrypt_viewkey(const crypto::chacha_key &key) + { + encrypt_viewkey(key); + } //----------------------------------------------------------------- account_base::account_base() { @@ -70,6 +136,16 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::set_null() { m_keys = account_keys(); + m_creation_timestamp = 0; + } + //----------------------------------------------------------------- + void account_base::deinit() + { + try{ + m_keys.get_device().disconnect(); + } catch (const std::exception &e){ + MERROR("Device disconnect exception: " << e.what()); + } } //----------------------------------------------------------------- void account_base::forget_spend_key() @@ -131,15 +207,24 @@ DISABLE_VS_WARNINGS(4244 4345) //----------------------------------------------------------------- void account_base::create_from_device(const std::string &device_name) { - hw::device &hwdev = hw::get_device(device_name); - m_keys.set_device(hwdev); hwdev.set_name(device_name); - MCDEBUG("ledger", "device type: "<<typeid(hwdev).name()); - hwdev.init(); - hwdev.connect(); - hwdev.get_public_address(m_keys.m_account_address); - hwdev.get_secret_keys(m_keys.m_view_secret_key, m_keys.m_spend_secret_key); + create_from_device(hwdev); + } + + void account_base::create_from_device(hw::device &hwdev) + { + m_keys.set_device(hwdev); + MCDEBUG("device", "device type: "<<typeid(hwdev).name()); + CHECK_AND_ASSERT_THROW_MES(hwdev.init(), "Device init failed"); + CHECK_AND_ASSERT_THROW_MES(hwdev.connect(), "Device connect failed"); + try { + CHECK_AND_ASSERT_THROW_MES(hwdev.get_public_address(m_keys.m_account_address), "Cannot get a device address"); + CHECK_AND_ASSERT_THROW_MES(hwdev.get_secret_keys(m_keys.m_view_secret_key, m_keys.m_spend_secret_key), "Cannot get device secret"); + } catch (const std::exception &e){ + hwdev.disconnect(); + throw; + } struct tm timestamp = {0}; timestamp.tm_year = 2014 - 1900; // year 2014 timestamp.tm_mon = 4 - 1; // month april @@ -157,7 +242,7 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey) { crypto::secret_key fake; - memset(&fake, 0, sizeof(fake)); + memset(&unwrap(unwrap(fake)), 0, sizeof(fake)); create_from_keys(address, fake, viewkey); } //----------------------------------------------------------------- diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index b5d119c46..021f84029 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -44,18 +44,29 @@ namespace cryptonote crypto::secret_key m_view_secret_key; std::vector<crypto::secret_key> m_multisig_keys; hw::device *m_device = &hw::get_device("default"); + crypto::chacha_iv m_encryption_iv; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(m_account_address) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_secret_key) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_secret_key) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_multisig_keys) + const crypto::chacha_iv default_iv{{0, 0, 0, 0, 0, 0, 0, 0}}; + KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(m_encryption_iv, default_iv) END_KV_SERIALIZE_MAP() account_keys& operator=(account_keys const&) = default; + void encrypt(const crypto::chacha_key &key); + void decrypt(const crypto::chacha_key &key); + void encrypt_viewkey(const crypto::chacha_key &key); + void decrypt_viewkey(const crypto::chacha_key &key); + hw::device& get_device() const ; void set_device( hw::device &hwdev) ; + + private: + void xor_with_key_stream(const crypto::chacha_key &key); }; /************************************************************************/ @@ -66,7 +77,8 @@ namespace cryptonote public: account_base(); crypto::secret_key generate(const crypto::secret_key& recovery_key = crypto::secret_key(), bool recover = false, bool two_random = false); - void create_from_device(const std::string &device_name) ; + void create_from_device(const std::string &device_name); + void create_from_device(hw::device &hwdev); void create_from_keys(const cryptonote::account_public_address& address, const crypto::secret_key& spendkey, const crypto::secret_key& viewkey); void create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey); bool make_multisig(const crypto::secret_key &view_secret_key, const crypto::secret_key &spend_secret_key, const crypto::public_key &spend_public_key, const std::vector<crypto::secret_key> &multisig_keys); @@ -77,6 +89,7 @@ namespace cryptonote hw::device& get_device() const {return m_keys.get_device();} void set_device( hw::device &hwdev) {m_keys.set_device(hwdev);} + void deinit(); uint64_t get_createtime() const { return m_creation_timestamp; } void set_createtime(uint64_t val) { m_creation_timestamp = val; } @@ -87,6 +100,11 @@ namespace cryptonote void forget_spend_key(); const std::vector<crypto::secret_key> &get_multisig_keys() const { return m_keys.m_multisig_keys; } + void encrypt_keys(const crypto::chacha_key &key) { m_keys.encrypt(key); } + void decrypt_keys(const crypto::chacha_key &key) { m_keys.decrypt(key); } + void encrypt_viewkey(const crypto::chacha_key &key) { m_keys.encrypt_viewkey(key); } + void decrypt_viewkey(const crypto::chacha_key &key) { m_keys.decrypt_viewkey(key); } + template <class t_archive> inline void serialize(t_archive &a, const unsigned int /*ver*/) { diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 5cd1709ab..eb73ab0ea 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -52,7 +52,7 @@ namespace cryptonote }; state m_state; - std::list<crypto::hash> m_needed_objects; + std::vector<crypto::hash> m_needed_objects; std::unordered_set<crypto::hash> m_requested_objects; uint64_t m_remote_blockchain_height; uint64_t m_last_response_height; @@ -67,15 +67,15 @@ namespace cryptonote switch (s) { case cryptonote_connection_context::state_before_handshake: - return "state_before_handshake"; + return "before_handshake"; case cryptonote_connection_context::state_synchronizing: - return "state_synchronizing"; + return "synchronizing"; case cryptonote_connection_context::state_standby: - return "state_standby"; + return "standby"; case cryptonote_connection_context::state_idle: - return "state_idle"; + return "idle"; case cryptonote_connection_context::state_normal: - return "state_normal"; + return "normal"; default: return "unknown"; } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index d4558ef7b..b0eabb0aa 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -47,7 +47,6 @@ #include "crypto/crypto.h" #include "crypto/hash.h" #include "misc_language.h" -#include "tx_extra.h" #include "ringct/rctTypes.h" #include "device/device.hpp" diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index 08a95d2e9..b18ef1c5c 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -67,7 +67,7 @@ namespace cryptonote { /* Cryptonote helper functions */ /************************************************************************/ //----------------------------------------------------------------------------------------------- - size_t get_min_block_size(uint8_t version) + size_t get_min_block_weight(uint8_t version) { if (version < 2) return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; @@ -86,7 +86,7 @@ namespace cryptonote { return CRYPTONOTE_MAX_TX_SIZE; } //----------------------------------------------------------------------------------------------- - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { static_assert(DIFFICULTY_TARGET_V2%60==0&&DIFFICULTY_TARGET_V1%60==0,"difficulty targets must be a multiple of 60"); const int target = version < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; const int target_minutes = target / 60; @@ -98,37 +98,37 @@ namespace cryptonote { base_reward = FINAL_SUBSIDY_PER_MINUTE*target_minutes; } - uint64_t full_reward_zone = get_min_block_size(version); + uint64_t full_reward_zone = get_min_block_weight(version); //make it soft - if (median_size < full_reward_zone) { - median_size = full_reward_zone; + if (median_weight < full_reward_zone) { + median_weight = full_reward_zone; } - if (current_block_size <= median_size) { + if (current_block_weight <= median_weight) { reward = base_reward; return true; } - if(current_block_size > 2 * median_size) { - MERROR("Block cumulative size is too big: " << current_block_size << ", expected less than " << 2 * median_size); + if(current_block_weight > 2 * median_weight) { + MERROR("Block cumulative weight is too big: " << current_block_weight << ", expected less than " << 2 * median_weight); return false; } - assert(median_size < std::numeric_limits<uint32_t>::max()); - assert(current_block_size < std::numeric_limits<uint32_t>::max()); + assert(median_weight < std::numeric_limits<uint32_t>::max()); + assert(current_block_weight < std::numeric_limits<uint32_t>::max()); uint64_t product_hi; // BUGFIX: 32-bit saturation bug (e.g. ARM7), the result was being // treated as 32-bit by default. - uint64_t multiplicand = 2 * median_size - current_block_size; - multiplicand *= current_block_size; + uint64_t multiplicand = 2 * median_weight - current_block_weight; + multiplicand *= current_block_weight; uint64_t product_lo = mul128(base_reward, multiplicand, &product_hi); uint64_t reward_hi; uint64_t reward_lo; - div128_32(product_hi, product_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); - div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); + div128_32(product_hi, product_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); + div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); assert(0 == reward_hi); assert(reward_lo < base_reward); @@ -162,10 +162,7 @@ namespace cryptonote { , account_public_address const & adr ) { - uint64_t address_prefix = nettype == TESTNET ? - (subaddress ? config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX) : nettype == STAGENET ? - (subaddress ? config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX) : - (subaddress ? config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX); + uint64_t address_prefix = subaddress ? get_config(nettype).CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : get_config(nettype).CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; return tools::base58::encode_addr(address_prefix, t_serializable_object_to_blob(adr)); } @@ -176,7 +173,7 @@ namespace cryptonote { , crypto::hash8 const & payment_id ) { - uint64_t integrated_address_prefix = nettype == TESTNET ? config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t integrated_address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; integrated_address iadr = { adr, payment_id @@ -201,15 +198,9 @@ namespace cryptonote { , std::string const & str ) { - uint64_t address_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; - uint64_t integrated_address_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; - uint64_t subaddress_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; + uint64_t address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; + uint64_t integrated_address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t subaddress_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; if (2 * sizeof(public_address_outer_blob) != str.size()) { diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index f59785021..c804a88fa 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -86,10 +86,10 @@ namespace cryptonote { /************************************************************************/ /* Cryptonote helper functions */ /************************************************************************/ - size_t get_min_block_size(uint8_t version); + size_t get_min_block_weight(uint8_t version); size_t get_max_block_size(); size_t get_max_tx_size(); - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); uint8_t get_account_address_checksum(const public_address_outer_blob& bl); uint8_t get_account_integrated_address_checksum(const public_integrated_address_outer_blob& bl); diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 143133163..0725a2bb8 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -295,7 +295,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +323,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,7 +337,7 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeSimpleBulletproof) + if (x.type == rct::RCTTypeBulletproof) a & x.p.pseudoOuts; } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 428be1c9c..e26aac76b 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -135,23 +135,41 @@ namespace cryptonote return false; } for (size_t n = 0; n < tx.rct_signatures.outPk.size(); ++n) + { + if (tx.vout[n].target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Unsupported output type in tx " << get_transaction_hash(tx)); + return false; + } rv.outPk[n].dest = rct::pk2rct(boost::get<txout_to_key>(tx.vout[n].target).key); + } if (!base_only) { - const bool bulletproof = rv.type == rct::RCTTypeFullBulletproof || rv.type == rct::RCTTypeSimpleBulletproof; + const bool bulletproof = rct::is_rct_bulletproof(rv.type); if (bulletproof) { - if (rv.p.bulletproofs.size() != tx.vout.size()) + if (rv.p.bulletproofs.size() != 1) { LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs size in tx " << get_transaction_hash(tx)); return false; } - for (size_t n = 0; n < rv.outPk.size(); ++n) + if (rv.p.bulletproofs[0].L.size() < 6) + { + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs L size in tx " << get_transaction_hash(tx)); + return false; + } + const size_t max_outputs = 1 << (rv.p.bulletproofs[0].L.size() - 6); + if (max_outputs < tx.vout.size()) { - rv.p.bulletproofs[n].V.resize(1); - rv.p.bulletproofs[n].V[0] = rv.outPk[n].mask; + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs max outputs in tx " << get_transaction_hash(tx)); + return false; } + const size_t n_amounts = tx.vout.size(); + CHECK_AND_ASSERT_MES(n_amounts == rv.outPk.size(), false, "Internal error filling out V"); + rv.p.bulletproofs[0].V.resize(n_amounts); + for (size_t i = 0; i < n_amounts; ++i) + rv.p.bulletproofs[0].V[i] = rct::scalarmultKey(rv.outPk[i].mask, rct::INV_EIGHT); } } } @@ -181,6 +199,16 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx) + { + std::stringstream ss; + ss << tx_blob; + binary_archive<false> ba(ss); + bool r = ::serialization::serialize_noeof(ba, tx); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction prefix from blob"); + return true; + } + //--------------------------------------------------------------- bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) { std::stringstream ss; @@ -201,15 +229,25 @@ namespace cryptonote { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); bool r = hwdev.generate_key_derivation(tx_public_key, ack.m_view_secret_key, recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + memcpy(&recv_derivation, rct::identity().bytes, sizeof(recv_derivation)); + } std::vector<crypto::key_derivation> additional_recv_derivations; for (size_t i = 0; i < additional_tx_public_keys.size(); ++i) { crypto::key_derivation additional_recv_derivation = AUTO_VAL_INIT(additional_recv_derivation); r = hwdev.generate_key_derivation(additional_tx_public_keys[i], ack.m_view_secret_key, additional_recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); - additional_recv_derivations.push_back(additional_recv_derivation); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); + } + else + { + additional_recv_derivations.push_back(additional_recv_derivation); + } } boost::optional<subaddress_receive_info> subaddr_recv_info = is_out_to_acc_precomp(subaddresses, out_key, recv_derivation, additional_recv_derivations, real_output_index,hwdev); @@ -318,6 +356,37 @@ namespace cryptonote return string_tools::get_xtype_from_string(amount, str_amount); } //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size) + { + if (tx.version < 2) + return blob_size; + const rct::rctSig &rv = tx.rct_signatures; + if (!rct::is_rct_bulletproof(rv.type)) + return blob_size; + const size_t n_outputs = tx.vout.size(); + if (n_outputs <= 2) + return blob_size; + const uint64_t bp_base = 368; + const size_t n_padded_outputs = rct::n_bulletproof_max_amounts(rv.p.bulletproofs); + size_t nlr = 0; + for (const auto &bp: rv.p.bulletproofs) + nlr += bp.L.size() * 2; + const size_t bp_size = 32 * (9 + nlr); + CHECK_AND_ASSERT_THROW_MES_L1(bp_base * n_padded_outputs >= bp_size, "Invalid bulletproof clawback"); + const uint64_t bp_clawback = (bp_base * n_padded_outputs - bp_size) * 4 / 5; + CHECK_AND_ASSERT_THROW_MES_L1(bp_clawback <= std::numeric_limits<uint64_t>::max() - blob_size, "Weight overflow"); + return blob_size + bp_clawback; + } + //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx) + { + std::ostringstream s; + binary_archive<true> a(s); + ::serialization::serialize(a, const_cast<transaction&>(tx)); + const cryptonote::blobdata blob = s.str(); + return get_transaction_weight(tx, blob.size()); + } + //--------------------------------------------------------------- bool get_tx_fee(const transaction& tx, uint64_t & fee) { if (tx.version > 1) @@ -376,6 +445,91 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + template<typename T> + static bool pick(binary_archive<true> &ar, std::vector<tx_extra_field> &fields, uint8_t tag) + { + std::vector<tx_extra_field>::iterator it; + while ((it = std::find_if(fields.begin(), fields.end(), [](const tx_extra_field &f) { return f.type() == typeid(T); })) != fields.end()) + { + bool r = ::do_serialize(ar, tag); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to serialize tx extra field"); + r = ::do_serialize(ar, boost::get<T>(*it)); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to serialize tx extra field"); + fields.erase(it); + } + return true; + } + //--------------------------------------------------------------- + bool sort_tx_extra(const std::vector<uint8_t>& tx_extra, std::vector<uint8_t> &sorted_tx_extra, bool allow_partial) + { + std::vector<tx_extra_field> tx_extra_fields; + + if(tx_extra.empty()) + { + sorted_tx_extra.clear(); + return true; + } + + std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); + std::istringstream iss(extra_str); + binary_archive<false> ar(iss); + + bool eof = false; + size_t processed = 0; + while (!eof) + { + tx_extra_field field; + bool r = ::do_serialize(ar, field); + if (!r) + { + MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + if (!allow_partial) + return false; + break; + } + tx_extra_fields.push_back(field); + processed = iss.tellg(); + + std::ios_base::iostate state = iss.rdstate(); + eof = (EOF == iss.peek()); + iss.clear(state); + } + if (!::serialization::check_stream_state(ar)) + { + MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + if (!allow_partial) + return false; + } + MTRACE("Sorted " << processed << "/" << tx_extra.size()); + + std::ostringstream oss; + binary_archive<true> nar(oss); + + // sort by: + if (!pick<tx_extra_pub_key>(nar, tx_extra_fields, TX_EXTRA_TAG_PUBKEY)) return false; + if (!pick<tx_extra_additional_pub_keys>(nar, tx_extra_fields, TX_EXTRA_TAG_ADDITIONAL_PUBKEYS)) return false; + if (!pick<tx_extra_nonce>(nar, tx_extra_fields, TX_EXTRA_NONCE)) return false; + if (!pick<tx_extra_merge_mining_tag>(nar, tx_extra_fields, TX_EXTRA_MERGE_MINING_TAG)) return false; + if (!pick<tx_extra_mysterious_minergate>(nar, tx_extra_fields, TX_EXTRA_MYSTERIOUS_MINERGATE_TAG)) return false; + if (!pick<tx_extra_padding>(nar, tx_extra_fields, TX_EXTRA_TAG_PADDING)) return false; + + // if not empty, someone added a new type and did not add a case above + if (!tx_extra_fields.empty()) + { + MERROR("tx_extra_fields not empty after sorting, someone forgot to add a case above"); + return false; + } + + std::string oss_str = oss.str(); + if (allow_partial && processed < tx_extra.size()) + { + MDEBUG("Appending unparsed data"); + oss_str += std::string((const char*)tx_extra.data() + processed, tx_extra.size() - processed); + } + sorted_tx_extra = std::vector<uint8_t>(oss_str.begin(), oss_str.end()); + return true; + } + //--------------------------------------------------------------- crypto::public_key get_tx_pub_key_from_extra(const std::vector<uint8_t>& tx_extra, size_t pk_index) { std::vector<tx_extra_field> tx_extra_fields; diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8a5296d5b..8d33b1ca4 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -31,6 +31,7 @@ #pragma once #include "blobdatatype.h" #include "cryptonote_basic_impl.h" +#include "tx_extra.h" #include "account.h" #include "subaddress_index.h" #include "include_base_utils.h" @@ -48,6 +49,7 @@ namespace cryptonote //--------------------------------------------------------------- void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h); crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); + bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx); @@ -64,6 +66,7 @@ namespace cryptonote } bool parse_tx_extra(const std::vector<uint8_t>& tx_extra, std::vector<tx_extra_field>& tx_extra_fields); + bool sort_tx_extra(const std::vector<uint8_t>& tx_extra, std::vector<uint8_t> &sorted_tx_extra, bool allow_partial = false); crypto::public_key get_tx_pub_key_from_extra(const std::vector<uint8_t>& tx_extra, size_t pk_index = 0); crypto::public_key get_tx_pub_key_from_extra(const transaction_prefix& tx, size_t pk_index = 0); crypto::public_key get_tx_pub_key_from_extra(const transaction& tx, size_t pk_index = 0); @@ -117,6 +120,8 @@ namespace cryptonote bool check_inputs_types_supported(const transaction& tx); bool check_outs_valid(const transaction& tx); bool parse_amount(uint64_t& amount, const std::string& str_amount); + uint64_t get_transaction_weight(const transaction &tx); + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size); bool check_money_overflow(const transaction& tx); bool check_outs_overflow(const transaction& tx); diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index ee5ec0596..a63a66976 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -220,6 +220,14 @@ namespace cryptonote */ uint64_t get_window_size() const { return window_size; } + struct Params { + uint8_t version; + uint8_t threshold; + uint64_t height; + time_t time; + Params(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), threshold(threshold), height(height), time(time) {} + }; + private: uint8_t get_block_version(uint64_t height) const; @@ -244,13 +252,6 @@ namespace cryptonote uint8_t original_version; uint64_t original_version_till_height; - struct Params { - uint8_t version; - uint8_t threshold; - uint64_t height; - time_t time; - Params(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), threshold(threshold), height(height), time(time) {} - }; std::vector<Params> heights; std::deque<uint8_t> versions; /* rolling window of the last N blocks' versions */ diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 2c777f5a2..d8ca2dd35 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -121,7 +121,8 @@ namespace cryptonote //----------------------------------------------------------------------------------------------------- miner::~miner() { - stop(); + try { stop(); } + catch (...) { /* ignore */ } } //----------------------------------------------------------------------------------------------------- bool miner::set_block_template(const block& bl, const difficulty_type& di, uint64_t height) @@ -198,8 +199,9 @@ namespace cryptonote { uint64_t total_hr = std::accumulate(m_last_hash_rates.begin(), m_last_hash_rates.end(), 0); float hr = static_cast<float>(total_hr)/static_cast<float>(m_last_hash_rates.size()); + const auto flags = std::cout.flags(); const auto precision = std::cout.precision(); - std::cout << "hashrate: " << std::setprecision(4) << std::fixed << hr << precision << ENDL; + std::cout << "hashrate: " << std::setprecision(4) << std::fixed << hr << std::setiosflags(flags) << std::setprecision(precision) << ENDL; } } m_last_hr_merge_time = misc_utils::get_tick_count(); @@ -328,6 +330,11 @@ namespace cryptonote LOG_PRINT_L0("Background mining controller thread started" ); } + if(get_ignore_battery()) + { + MINFO("Ignoring battery"); + } + return true; } //----------------------------------------------------------------------------------------------------- @@ -485,7 +492,7 @@ namespace cryptonote { //we lucky! ++m_config.current_extra_message_index; - MGINFO_GREEN("Found block for difficulty: " << local_diff); + MGINFO_GREEN("Found block " << get_block_hash(b) << " at height " << height << " for difficulty: " << local_diff); if(!m_phandler->handle_block_found(b)) { --m_config.current_extra_message_index; @@ -630,7 +637,7 @@ namespace cryptonote boost::tribool battery_powered(on_battery_power()); if(!indeterminate( battery_powered )) { - on_ac_power = !battery_powered; + on_ac_power = !(bool)battery_powered; } } diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 3e64073dd..c62eeb738 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -30,6 +30,7 @@ #pragma once +#include <stdexcept> #include <string> #include <boost/uuid/uuid.hpp> @@ -65,9 +66,11 @@ #define FEE_PER_KB_OLD ((uint64_t)10000000000) // pow(10, 10) #define FEE_PER_KB ((uint64_t)2000000000) // 2 * pow(10, 9) +#define FEE_PER_BYTE ((uint64_t)300000) #define DYNAMIC_FEE_PER_KB_BASE_FEE ((uint64_t)2000000000) // 2 * pow(10,9) #define DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD ((uint64_t)10000000000000) // 10 * pow(10,12) #define DYNAMIC_FEE_PER_KB_BASE_FEE_V5 ((uint64_t)2000000000 * (uint64_t)CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 / CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5) +#define DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT ((uint64_t)3000) #define ORPHANED_BLOCKS_MAX_COUNT 100 @@ -133,13 +136,17 @@ #define HF_VERSION_DYNAMIC_FEE 4 #define HF_VERSION_MIN_MIXIN_4 6 #define HF_VERSION_MIN_MIXIN_6 7 +#define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 +#define HF_VERSION_PER_BYTE_FEE 8 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 #define HASH_OF_HASHES_STEP 256 -#define DEFAULT_TXPOOL_MAX_SIZE 648000000ull // 3 days at 300000, in bytes +#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes + +#define BULLETPROOF_MAX_OUTPUTS 16 // New constants are intended to go here namespace config @@ -203,4 +210,60 @@ namespace cryptonote FAKECHAIN, UNDEFINED = 255 }; + struct config_t + { + uint64_t const CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; + uint64_t const CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t const CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; + uint16_t const P2P_DEFAULT_PORT; + uint16_t const RPC_DEFAULT_PORT; + uint16_t const ZMQ_RPC_DEFAULT_PORT; + boost::uuids::uuid const NETWORK_ID; + std::string const GENESIS_TX; + uint32_t const GENESIS_NONCE; + }; + inline const config_t& get_config(network_type nettype) + { + static const config_t mainnet = { + ::config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::P2P_DEFAULT_PORT, + ::config::RPC_DEFAULT_PORT, + ::config::ZMQ_RPC_DEFAULT_PORT, + ::config::NETWORK_ID, + ::config::GENESIS_TX, + ::config::GENESIS_NONCE + }; + static const config_t testnet = { + ::config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::testnet::P2P_DEFAULT_PORT, + ::config::testnet::RPC_DEFAULT_PORT, + ::config::testnet::ZMQ_RPC_DEFAULT_PORT, + ::config::testnet::NETWORK_ID, + ::config::testnet::GENESIS_TX, + ::config::testnet::GENESIS_NONCE + }; + static const config_t stagenet = { + ::config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::stagenet::P2P_DEFAULT_PORT, + ::config::stagenet::RPC_DEFAULT_PORT, + ::config::stagenet::ZMQ_RPC_DEFAULT_PORT, + ::config::stagenet::NETWORK_ID, + ::config::stagenet::GENESIS_TX, + ::config::stagenet::GENESIS_NONCE + }; + switch (nettype) + { + case MAINNET: return mainnet; + case TESTNET: return testnet; + case STAGENET: return stagenet; + case FAKECHAIN: return mainnet; + default: throw std::runtime_error("Invalid network type"); + } + }; } diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 72844db66..231489fdb 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -41,12 +41,6 @@ set(cryptonote_core_private_headers tx_pool.h cryptonote_tx_utils.h) -if(PER_BLOCK_CHECKPOINT) - set(Blocks "blocks") -else() - set(Blocks "") -endif() - monero_private_headers(cryptonote_core ${cryptonote_core_private_headers}) monero_add_library(cryptonote_core @@ -69,5 +63,4 @@ target_link_libraries(cryptonote_core ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE - ${Blocks} ${EXTRA_LIBRARIES}) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index fbb2a2223..77b6d0b69 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -52,9 +52,7 @@ #include "cryptonote_core.h" #include "ringct/rctSigs.h" #include "common/perf_timer.h" -#if defined(PER_BLOCK_CHECKPOINT) -#include "blocks/blocks.h" -#endif +#include "common/notify.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -109,6 +107,12 @@ static const struct { // version 7 starts from block 1546000, which is on or around the 6th of April, 2018. Fork time finalised on 2018-03-17. { 7, 1546000, 0, 1521303150 }, + + // version 8 starts from block 1685555, which is on or around the 18th of October, 2018. Fork time finalised on 2018-09-02. + { 8, 1685555, 0, 1535889547 }, + + // version 9 starts from block 1686275, which is on or around the 19th of October, 2018. Fork time finalised on 2018-09-02. + { 9, 1686275, 0, 1535889548 }, }; static const uint64_t mainnet_hard_fork_version_1_till = 1009826; @@ -131,7 +135,8 @@ static const struct { { 6, 971400, 0, 1501709789 }, { 7, 1057027, 0, 1512211236 }, - { 8, 1057058, 0, 1515967497 }, + { 8, 1057058, 0, 1533211200 }, + { 9, 1057778, 0, 1533297600 }, }; static const uint64_t testnet_hard_fork_version_1_till = 624633; @@ -151,14 +156,17 @@ static const struct { { 5, 35000, 0, 1521360000 }, { 6, 36000, 0, 1521480000 }, { 7, 37000, 0, 1521600000 }, + { 8, 176456, 0, 1537821770 }, + { 9, 177176, 0, 1537821771 }, }; //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : - m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), m_current_block_cumul_sz_median(0), - m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false), + m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), + m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), m_difficulty_for_next_block_top_hash(crypto::null_hash), - m_difficulty_for_next_block(1) + m_difficulty_for_next_block(1), + m_btc_valid(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -242,6 +250,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke MDEBUG("Additional outputs needed: " << absolute_offsets.size() - outputs.size()); std::vector < uint64_t > add_offsets; std::vector<output_data_t> add_outputs; + add_outputs.reserve(absolute_offsets.size() - outputs.size()); for (size_t i = outputs.size(); i < absolute_offsets.size(); i++) add_offsets.push_back(absolute_offsets[i]); try @@ -329,9 +338,12 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options) +bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options, difficulty_type fixed_difficulty, const GetCheckpointsCallback& get_checkpoints/* = nullptr*/) { LOG_PRINT_L3("Blockchain::" << __func__); + + CHECK_AND_ASSERT_MES(nettype != FAKECHAIN || test_options, false, "fake chain network type used without options"); + CRITICAL_REGION_LOCAL(m_tx_pool); CRITICAL_REGION_LOCAL1(m_blockchain_lock); @@ -351,6 +363,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_nettype = test_options != NULL ? FAKECHAIN : nettype; m_offline = offline; + m_fixed_difficulty = fixed_difficulty; if (m_hardfork == nullptr) { if (m_nettype == FAKECHAIN || m_nettype == STAGENET) @@ -393,18 +406,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline MINFO("Blockchain not loaded, generating genesis block."); block bl = boost::value_initialized<block>(); block_verification_context bvc = boost::value_initialized<block_verification_context>(); - if (m_nettype == TESTNET) - { - generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else if (m_nettype == STAGENET) - { - generate_genesis_block(bl, config::stagenet::GENESIS_TX, config::stagenet::GENESIS_NONCE); - } - else - { - generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); - } + generate_genesis_block(bl, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); add_new_block(bl, bvc); CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); } @@ -437,14 +439,14 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline #if defined(PER_BLOCK_CHECKPOINT) if (m_nettype != FAKECHAIN) - load_compiled_in_block_hashes(); + load_compiled_in_block_hashes(get_checkpoints); #endif MINFO("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block()); m_db->block_txn_stop(); uint64_t num_popped_blocks = 0; - while (true) + while (!m_db->is_read_only()) { const uint64_t top_height = m_db->height() - 1; const crypto::hash top_id = m_db->top_block_hash(); @@ -490,7 +492,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); } - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return true; } //------------------------------------------------------------------ @@ -639,8 +641,9 @@ block Blockchain::pop_block_from_blockchain() m_blocks_txs_check.clear(); m_check_txin_table.clear(); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); + invalidate_block_template_cache(); return popped_block; } @@ -651,12 +654,13 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) CRITICAL_REGION_LOCAL(m_blockchain_lock); m_timestamps_and_difficulties_height = 0; m_alternative_chains.clear(); + invalidate_block_template_cache(); m_db->reset(); m_hardfork->init(); block_verification_context bvc = boost::value_initialized<block_verification_context>(); add_new_block(b, bvc); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ @@ -805,6 +809,11 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph // less blocks than desired if there aren't enough. difficulty_type Blockchain::get_difficulty_for_next_block() { + if (m_fixed_difficulty) + { + return m_db->height() ? m_fixed_difficulty : 1; + } + LOG_PRINT_L3("Blockchain::" << __func__); crypto::hash top_hash = get_tail_id(); @@ -827,7 +836,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() // then when the next block difficulty is queried, push the latest height data and // pop the oldest one from the list. This only requires 1x read per height instead // of doing 735 (DIFFICULTY_BLOCKS_COUNT). - if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1)) + if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1) && m_timestamps.size() >= DIFFICULTY_BLOCKS_COUNT) { uint64_t index = height - 1; m_timestamps.push_back(m_db->get_block_timestamp(index)); @@ -850,6 +859,11 @@ difficulty_type Blockchain::get_difficulty_for_next_block() timestamps.clear(); difficulties.clear(); + if (height > offset) + { + timestamps.reserve(height - offset); + difficulties.reserve(height - offset); + } for (; offset < height; offset++) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -869,6 +883,15 @@ difficulty_type Blockchain::get_difficulty_for_next_block() return diff; } //------------------------------------------------------------------ +std::vector<time_t> Blockchain::get_last_block_timestamps(unsigned int blocks) const +{ + std::vector<time_t> timestamps(blocks); + uint64_t height = m_db->height(); + while (blocks--) + timestamps[blocks] = m_db->get_block_timestamp(height - blocks - 1); + return timestamps; +} +//------------------------------------------------------------------ // This function removes blocks from the blockchain until it gets to the // position where the blockchain switch started and then re-adds the blocks // that had been removed. @@ -1011,6 +1034,11 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash:: // an alternate chain. difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const { + if (m_fixed_difficulty) + { + return m_db->height() ? m_fixed_difficulty : 1; + } + LOG_PRINT_L3("Blockchain::" << __func__); std::vector<uint64_t> timestamps; std::vector<difficulty_type> cumulative_difficulties; @@ -1104,7 +1132,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) } //------------------------------------------------------------------ // This function validates the miner transaction reward -bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) { LOG_PRINT_L3("Blockchain::" << __func__); //validate reward @@ -1122,11 +1150,11 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } } - std::vector<size_t> last_blocks_sizes; - get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward, version)) + std::vector<size_t> last_blocks_weights; + get_last_n_blocks_weights(last_blocks_weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_weights), cumulative_block_weight, already_generated_coins, base_reward, version)) { - MERROR_VER("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + MERROR_VER("block weight " << cumulative_block_weight << " is bigger than allowed for this blockchain"); return false; } if(base_reward + fee < money_in_use) @@ -1156,8 +1184,8 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return true; } //------------------------------------------------------------------ -// get the block sizes of the last <count> blocks, and return by reference <sz>. -void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const +// get the block weights of the last <count> blocks, and return by reference <sz>. +void Blockchain::get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1168,25 +1196,26 @@ void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) return; m_db->block_txn_start(true); - // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) + // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); + weights.reserve(weights.size() + h - start_offset); for(size_t i = start_offset; i < h; i++) { - sz.push_back(m_db->get_block_size(i)); + weights.push_back(m_db->get_block_weight(i)); } m_db->block_txn_stop(); } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_limit() const +uint64_t Blockchain::get_current_cumulative_block_weight_limit() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_limit; + return m_current_block_cumul_weight_limit; } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_median() const +uint64_t Blockchain::get_current_cumulative_block_weight_median() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_median; + return m_current_block_cumul_weight_median; } //------------------------------------------------------------------ //TODO: This function only needed minor modification to work with BlockchainDB, @@ -1203,11 +1232,28 @@ uint64_t Blockchain::get_current_cumulative_blocksize_median() const bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) { LOG_PRINT_L3("Blockchain::" << __func__); - size_t median_size; + size_t median_weight; uint64_t already_generated_coins; + uint64_t pool_cookie; CRITICAL_REGION_BEGIN(m_blockchain_lock); height = m_db->height(); + if (m_btc_valid) { + // The pool cookie is atomic. The lack of locking is OK, as if it changes + // just as we compare it, we'll just use a slightly old template, but + // this would be the case anyway if we'd lock, and the change happened + // just after the block template was created + if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce && m_btc_pool_cookie == m_tx_pool.cookie()) { + MDEBUG("Using cached template"); + m_btc.timestamp = time(NULL); // update timestamp unconditionally + b = m_btc; + diffic = m_btc_difficulty; + expected_reward = m_btc_expected_reward; + return true; + } + MDEBUG("Not using cached template: address " << (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address))) << ", nonce " << (m_btc_nonce == ex_nonce) << ", cookie " << (m_btc_pool_cookie == m_tx_pool.cookie())); + invalidate_block_template_cache(); + } b.major_version = m_hardfork->get_current_version(); b.minor_version = m_hardfork->get_ideal_version(); @@ -1223,19 +1269,20 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m diffic = get_difficulty_for_next_block(); CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead."); - median_size = m_current_block_cumul_sz_limit / 2; + median_weight = m_current_block_cumul_weight_limit / 2; already_generated_coins = m_db->get_block_already_generated_coins(height - 1); CRITICAL_REGION_END(); - size_t txs_size; + size_t txs_weight; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee, expected_reward, m_hardfork->get_current_version())) + if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, m_hardfork->get_current_version())) { return false; } + pool_cookie = m_tx_pool.cookie(); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - size_t real_txs_size = 0; + size_t real_txs_weight = 0; uint64_t real_fee = 0; CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); for(crypto::hash &cur_hash: b.tx_hashes) @@ -1247,11 +1294,11 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m continue; } tx_memory_pool::tx_details &cur_tx = cur_res->second; - real_txs_size += cur_tx.blob_size; + real_txs_weight += cur_tx.weight; real_fee += cur_tx.fee; - if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) + if (cur_tx.weight != get_transaction_weight(cur_tx.tx)) { - LOG_ERROR("Creating block template: error: invalid transaction size"); + LOG_ERROR("Creating block template: error: invalid transaction weight"); } if (cur_tx.tx.version == 1) { @@ -1273,9 +1320,9 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } } } - if (txs_size != real_txs_size) + if (txs_weight != real_txs_weight) { - LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + LOG_ERROR("Creating block template: error: wrongly calculated transaction weight"); } if (fee != real_fee) { @@ -1283,71 +1330,73 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } CRITICAL_REGION_END(); MDEBUG("Creating block template: height " << height << - ", median size " << median_size << + ", median weight " << median_weight << ", already generated coins " << already_generated_coins << - ", transaction size " << txs_size << + ", transaction weight " << txs_weight << ", fee " << fee); #endif /* - two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know - block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + two-phase miner transaction generation: we don't know exact block weight until we prepare block, but we don't know reward until we know + block weight, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block weight */ - //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob weight uint8_t hf_version = m_hardfork->get_current_version(); size_t max_outs = hf_version >= 4 ? 1 : 11; - bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + bool r = construct_miner_tx(height, median_weight, already_generated_coins, txs_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, first chance"); - size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); + size_t cumulative_weight = txs_weight + get_transaction_weight(b.miner_tx); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << - ", cumulative size " << cumulative_size); + MDEBUG("Creating block template: miner tx weight " << get_transaction_weight(b.miner_tx) << + ", cumulative weight " << cumulative_weight); #endif for (size_t try_count = 0; try_count != 10; ++try_count) { - r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + r = construct_miner_tx(height, median_weight, already_generated_coins, cumulative_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, second chance"); - size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); - if (coinbase_blob_size > cumulative_size - txs_size) + size_t coinbase_weight = get_transaction_weight(b.miner_tx); + if (coinbase_weight > cumulative_weight - txs_weight) { - cumulative_size = txs_size + coinbase_blob_size; + cumulative_weight = txs_weight + coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is greater than before"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is greater than before"); #endif continue; } - if (coinbase_blob_size < cumulative_size - txs_size) + if (coinbase_weight < cumulative_weight - txs_weight) { - size_t delta = cumulative_size - txs_size - coinbase_blob_size; + size_t delta = cumulative_weight - txs_weight - coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << txs_size + coinbase_blob_size << + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << txs_weight + coinbase_weight << " is less than before, adding " << delta << " zero bytes"); #endif b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight + 1 == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " + 1 is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_weight MDEBUG("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1); - cumulative_size += delta - 1; + cumulative_weight += delta - 1; continue; } MDEBUG("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count); } } - CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is now good"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is now good"); #endif + + cache_block_template(b, miner_address, ex_nonce, diffic, expected_reward, pool_cookie); return true; } LOG_ERROR("Failed to create_block_template with " << 10 << " tries"); @@ -1367,6 +1416,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + timestamps.reserve(timestamps.size() + start_top_height - stop_offset); while (start_top_height != stop_offset) { timestamps.push_back(m_db->get_block_timestamp(start_top_height)); @@ -1566,7 +1616,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1580,7 +1630,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std:: for(const auto& blk : blocks) { - std::list<crypto::hash> missed_ids; + std::vector<crypto::hash> missed_ids; get_transactions_blobs(blk.second.tx_hashes, txs, missed_ids); CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); } @@ -1588,14 +1638,16 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std:: return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - if(start_offset >= m_db->height()) + const uint64_t height = m_db->height(); + if(start_offset >= height) return false; - for(size_t i = start_offset; i < start_offset + count && i < m_db->height();i++) + blocks.reserve(blocks.size() + height - start_offset); + for(size_t i = start_offset; i < start_offset + count && i < height;i++) { blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(i), block())); if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) @@ -1620,17 +1672,20 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO CRITICAL_REGION_LOCAL(m_blockchain_lock); m_db->block_txn_start(true); rsp.current_blockchain_height = get_current_blockchain_height(); - std::list<std::pair<cryptonote::blobdata,block>> blocks; + std::vector<std::pair<cryptonote::blobdata,block>> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); - for (const auto& bl: blocks) + for (auto& bl: blocks) { - std::list<crypto::hash> missed_tx_ids; - std::list<cryptonote::blobdata> txs; + std::vector<crypto::hash> missed_tx_ids; + std::vector<cryptonote::blobdata> txs; + + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); // FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids // is for missed blocks, not missed transactions as well. - get_transactions_blobs(bl.second.tx_hashes, txs, missed_tx_ids); + get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids); if (missed_tx_ids.size() != 0) { @@ -1642,35 +1697,28 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO // append missed transaction hashes to response missed_ids field, // as done below if any standalone transactions were requested // and missed. - rsp.missed_ids.splice(rsp.missed_ids.end(), missed_tx_ids); - m_db->block_txn_stop(); + rsp.missed_ids.insert(rsp.missed_ids.end(), missed_tx_ids.begin(), missed_tx_ids.end()); + m_db->block_txn_stop(); return false; } - rsp.blocks.push_back(block_complete_entry()); - block_complete_entry& e = rsp.blocks.back(); //pack block - e.block = bl.first; - //pack transactions - for (const cryptonote::blobdata& tx: txs) - e.txs.push_back(tx); - } - //get another transactions, if need - std::list<cryptonote::blobdata> txs; - get_transactions_blobs(arg.txs, txs, rsp.missed_ids); - //pack aside transactions - for (const auto& tx: txs) - rsp.txs.push_back(tx); + e.block = std::move(bl.first); + } + //get and pack other transactions, if needed + std::vector<cryptonote::blobdata> txs; + get_transactions_blobs(arg.txs, rsp.txs, rsp.missed_ids); m_db->block_txn_stop(); return true; } //------------------------------------------------------------------ -bool Blockchain::get_alternative_blocks(std::list<block>& blocks) const +bool Blockchain::get_alternative_blocks(std::vector<block>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + blocks.reserve(m_alternative_chains.size()); for (const auto& alt_bl: m_alternative_chains) { blocks.push_back(alt_bl.second.bl); @@ -1687,17 +1735,6 @@ size_t Blockchain::get_alternative_blocks_count() const //------------------------------------------------------------------ // This function adds the output specified by <amount, i> to the result_outs container // unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; -} - uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const { uint64_t num_outs = m_db->get_num_outputs(amount); @@ -1715,74 +1752,6 @@ uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const return num_outs; } -std::vector<uint64_t> Blockchain::get_random_outputs(uint64_t amount, uint64_t count) const -{ - uint64_t num_outs = get_num_mature_outputs(amount); - - std::vector<uint64_t> indices; - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if tx is unlocked, add output to indices - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - else - { - // while we still need more mixins - while (indices.size() < count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - - return indices; -} - crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_index) const { output_data_t data = m_db->get_output_key(amount, global_index); @@ -1790,184 +1759,28 @@ crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_i } //------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - for (uint64_t amount : req.amounts) - { - // create outs_for_amount struct and populate amount field - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); - result_outs.amount = amount; - - std::vector<uint64_t> indices = get_random_outputs(amount, req.outs_count); - - for (auto i : indices) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oe = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - - oe.global_amount_index = i; - oe.out_key = get_output_key(amount, i); - } - } - return true; -} -//------------------------------------------------------------------ -// This function adds the ringct output at index i to the list -// unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry& oen = *outs.insert(outs.end(), COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry()); - oen.amount = amount; - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; - oen.commitment = data.commitment; -} -//------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const +bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - auto num_outs = m_db->get_num_outputs(0); - // ensure we don't include outputs that aren't yet eligible to be used - // outpouts are sorted by height - while (num_outs > 0) - { - const tx_out_index toi = m_db->get_output_tx_and_index(0, num_outs - 1); - const uint64_t height = m_db->get_tx_block_height(toi.first); - if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height()) - break; - --num_outs; - } - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= req.outs_count) + res.outs.clear(); + res.outs.reserve(req.outputs.size()); + try { - for (uint64_t i = 0; i < num_outs; i++) + for (const auto &i: req.outputs) { // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); + const output_data_t od = m_db->get_output_key(i.amount, i.index); + tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); + bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); - // if tx is unlocked, add output to result_outs - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } + res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); } } - else + catch (const std::exception &e) { - // while we still need more mixins - while (res.outs.size() < req.outs_count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - - if (res.outs.size() < req.outs_count) return false; -#if 0 - // if we do not have enough RCT inputs, we can pick from the non RCT ones - // which will have a zero mask - if (res.outs.size() < req.outs_count) - { - LOG_PRINT_L0("Out of RCT inputs (" << res.outs.size() << "/" << req.outs_count << "), using regular ones"); - - // TODO: arbitrary selection, needs better - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req2 = AUTO_VAL_INIT(req2); - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response res2 = AUTO_VAL_INIT(res2); - req2.outs_count = req.outs_count - res.outs.size(); - static const uint64_t amounts[] = {1, 10, 20, 50, 100, 200, 500, 1000, 10000}; - for (uint64_t a: amounts) - req2.amounts.push_back(a); - if (!get_random_outs_for_amounts(req2, res2)) - return false; - - // pick random ones from there - while (res.outs.size() < req.outs_count) - { - int list_idx = rand() % (sizeof(amounts)/sizeof(amounts[0])); - if (!res2.outs[list_idx].outs.empty()) - { - const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry oe = res2.outs[list_idx].outs.back(); - res2.outs[list_idx].outs.pop_back(); - add_out_to_get_rct_random_outs(res.outs, res2.outs[list_idx].amount, oe.global_amount_index); - } - } - } -#endif - - return true; -} -//------------------------------------------------------------------ -bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - res.outs.clear(); - res.outs.reserve(req.outputs.size()); - for (const auto &i: req.outputs) - { - // get tx_hash, tx_out_index from DB - const output_data_t od = m_db->get_output_key(i.amount, i.index); - tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); - bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); - - res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); } return true; } @@ -1983,14 +1796,14 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint //------------------------------------------------------------------ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const { - // rct outputs don't exist before v3 + // rct outputs don't exist before v4 if (amount == 0) { switch (m_nettype) { - case STAGENET: start_height = stagenet_hard_forks[2].height; break; - case TESTNET: start_height = testnet_hard_forks[2].height; break; - case MAINNET: start_height = mainnet_hard_forks[2].height; break; + case STAGENET: start_height = stagenet_hard_forks[3].height; break; + case TESTNET: start_height = testnet_hard_forks[3].height; break; + case MAINNET: start_height = mainnet_hard_forks[3].height; break; default: return false; } } @@ -1998,11 +1811,35 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, start_height = 0; base = 0; + if (to_height > 0 && to_height < from_height) + return false; + const uint64_t real_start_height = start_height; if (from_height > start_height) start_height = from_height; - return m_db->get_output_distribution(amount, start_height, to_height, distribution, base); + distribution.clear(); + uint64_t db_height = m_db->height(); + if (db_height == 0) + return false; + if (to_height == 0) + to_height = db_height - 1; + if (start_height >= db_height || to_height >= db_height) + return false; + if (amount == 0) + { + std::vector<uint64_t> heights; + heights.reserve(to_height + 1 - start_height); + for (uint64_t h = start_height; h <= to_height; ++h) + heights.push_back(h); + distribution = m_db->get_block_cumulative_rct_outputs(heights); + base = 0; + return true; + } + else + { + return m_db->get_output_distribution(amount, start_height, to_height, distribution, base); + } } //------------------------------------------------------------------ // This function takes a list of block hashes from another node @@ -2015,9 +1852,9 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc // make sure the request includes at least the genesis block, otherwise // how can we expect to sync from the client that the block list came from? - if(!qblock_ids.size() /*|| !req.m_total_height*/) + if(!qblock_ids.size()) { - MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); + MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << ", dropping connection"); return false; } @@ -2083,6 +1920,9 @@ uint64_t Blockchain::block_difficulty(uint64_t i) const return 0; } //------------------------------------------------------------------ +template<typename T> void reserve_container(std::vector<T> &v, size_t N) { v.reserve(N); } +template<typename T> void reserve_container(std::list<T> &v, size_t N) { } +//------------------------------------------------------------------ //TODO: return type should be void, throw on exception // alternatively, return true only if no blocks missed template<class t_ids_container, class t_blocks_container, class t_missed_container> @@ -2091,20 +1931,24 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(blocks, block_ids.size()); for (const auto& block_hash : block_ids) { try { - blocks.push_back(std::make_pair(m_db->get_block_blob(block_hash), block())); - if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) + uint64_t height = 0; + if (m_db->block_exists(block_hash, &height)) { - LOG_ERROR("Invalid block"); - return false; + blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(height), block())); + if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) + { + LOG_ERROR("Invalid block: " << block_hash); + blocks.pop_back(); + missed_bs.push_back(block_hash); + } } - } - catch (const BLOCK_DNE& e) - { - missed_bs.push_back(block_hash); + else + missed_bs.push_back(block_hash); } catch (const std::exception& e) { @@ -2122,6 +1966,7 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2148,6 +1993,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2176,7 +2022,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container // Find the split point between us and foreign blockchain and return // (by reference) the most recent common block hash along with up to // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. -bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::list<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2190,6 +2036,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc m_db->block_txn_start(true); current_height = get_current_blockchain_height(); size_t count = 0; + hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); @@ -2205,7 +2052,8 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc CRITICAL_REGION_LOCAL(m_blockchain_lock); bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height); - resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); + if (result) + resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); return result; } @@ -2214,7 +2062,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height <req_start_block>), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2240,18 +2088,28 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons m_db->block_txn_start(true); total_height = get_current_blockchain_height(); size_t count = 0, size = 0; - for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) + blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); + for(uint64_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); - blocks.back().first = m_db->get_block_blob_from_height(i); + blocks.back().first.first = m_db->get_block_blob_from_height(i); block b; - CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first, b), false, "internal error, invalid block"); - std::list<crypto::hash> mis; - get_transactions_blobs(b.tx_hashes, blocks.back().second, mis, pruned); + CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first.first, b), false, "internal error, invalid block"); + blocks.back().first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash; + std::vector<crypto::hash> mis; + std::vector<cryptonote::blobdata> txs; + get_transactions_blobs(b.tx_hashes, txs, mis, pruned); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); - size += blocks.back().first.size(); - for (const auto &t: blocks.back().second) + size += blocks.back().first.first.size(); + for (const auto &t: txs) size += t.size(); + + CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size(), false, "mismatched sizes of b.tx_hashes and txs"); + blocks.back().second.reserve(txs.size()); + for (size_t i = 0; i < txs.size(); ++i) + { + blocks.back().second.push_back(std::make_pair(b.tx_hashes[i], std::move(txs[i]))); + } } m_db->block_txn_stop(); return true; @@ -2282,19 +2140,19 @@ bool Blockchain::have_block(const crypto::hash& id) const if(m_db->block_exists(id)) { - LOG_PRINT_L3("block exists in main chain"); + LOG_PRINT_L2("block " << id << " found in main chain"); return true; } if(m_alternative_chains.count(id)) { - LOG_PRINT_L3("block found in m_alternative_chains"); + LOG_PRINT_L2("block " << id << " found in m_alternative_chains"); return true; } if(m_invalid_blocks.count(id)) { - LOG_PRINT_L3("block found in m_invalid_blocks"); + LOG_PRINT_L2("block " << id << " found in m_invalid_blocks"); return true; } @@ -2454,7 +2312,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh if(m_show_time_stats) { size_t ring_size = !tx.vin.empty() && tx.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(tx.vin[0]).key_offsets.size() : 0; - MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx)); + MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx) << " W: " << get_transaction_weight(tx)); } if (!res) return false; @@ -2511,12 +2369,27 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context // from v8, allow bulletproofs if (hf_version < 8) { - const bool bulletproof = tx.rct_signatures.type == rct::RCTTypeFullBulletproof || tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof; - if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) - { - MERROR("Bulletproofs are not allowed before v8"); - tvc.m_invalid_output = true; - return false; + if (tx.version >= 2) { + const bool bulletproof = rct::is_rct_bulletproof(tx.rct_signatures.type); + if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) + { + MERROR_VER("Bulletproofs are not allowed before v8"); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v9, forbid borromean range proofs + if (hf_version > 8) { + if (tx.version >= 2) { + const bool borromean = rct::is_rct_borromean(tx.rct_signatures.type); + if (borromean) + { + MERROR_VER("Borromean range proofs are not allowed after v8"); + tvc.m_invalid_output = true; + return false; + } } } @@ -2545,7 +2418,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr rv.message = rct::hash2rct(tx_prefix_hash); // mixRing - full and simple store it in opposite ways - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys[0].size()); @@ -2560,7 +2433,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2579,14 +2452,14 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } // II - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { rv.p.MGs.resize(1); rv.p.MGs[0].II.resize(tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2628,7 +2501,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { size_t n_unmixable = 0, n_mixable = 0; size_t mixin = std::numeric_limits<size_t>::max(); - const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; + const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_10 ? 10 : hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; for (const auto& txin : tx.vin) { // non txin_to_key inputs will be rejected below @@ -2657,6 +2530,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } + if (hf_version >= HF_VERSION_MIN_MIXIN_10 && mixin != 10) + { + MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); + tvc.m_low_mixin = true; + return false; + } + if (mixin < min_mixin) { if (n_unmixable == 0) @@ -2723,7 +2603,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - const auto waiter_guard = epee::misc_utils::create_scope_leave_handler([&]() { waiter.wait(); }); + const auto waiter_guard = epee::misc_utils::create_scope_leave_handler([&]() { waiter.wait(&tpool); }); int threads = tpool.get_max_concurrency(); for (const auto& txin : tx.vin) @@ -2785,7 +2665,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { // ND: Speedup // 1. Thread ring signature verification if possible. - tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index]))); + tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index])), true); } else { @@ -2809,7 +2689,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, sig_index++; } if (tx.version == 1 && threads > 1) - waiter.wait(); + waiter.wait(&tpool); if (tx.version == 1) { @@ -2852,7 +2732,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, return false; } case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -2902,7 +2782,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (!rct::verRctSimple(rv, false)) + if (!rct::verRctNonSemanticsSimple(rv)) { MERROR_VER("Failed to check ringct signatures!"); return false; @@ -2910,7 +2790,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, break; } case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -2973,6 +2852,22 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, MERROR_VER("Unsupported rct type: " << rv.type); return false; } + + // for bulletproofs, check they're only multi-output after v8 + if (rct::is_rct_bulletproof(rv.type)) + { + if (hf_version < 8) + { + for (const rct::Bulletproof &proof: rv.p.bulletproofs) + { + if (proof.V.size() > 1) + { + MERROR_VER("Multi output bulletproofs are invalid before v8"); + return false; + } + } + } + } } return true; } @@ -2981,6 +2876,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature>& sig, uint64_t &result) { std::vector<const crypto::public_key *> p_output_keys; + p_output_keys.reserve(pubkeys.size()); for (auto &key : pubkeys) { // rct::key and crypto::public_key have the same structure, avoid object ctor/memcpy @@ -2991,7 +2887,7 @@ void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const } //------------------------------------------------------------------ -static uint64_t get_fee_quantization_mask() +uint64_t Blockchain::get_fee_quantization_mask() { static uint64_t mask = 0; if (mask == 0) @@ -3004,16 +2900,27 @@ static uint64_t get_fee_quantization_mask() } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version) +uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version) { - const uint64_t min_block_size = get_min_block_size(version); - const uint64_t fee_per_kb_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; + const uint64_t min_block_weight = get_min_block_weight(version); + if (median_block_weight < min_block_weight) + median_block_weight = min_block_weight; + uint64_t hi, lo; + + if (version >= HF_VERSION_PER_BYTE_FEE) + { + lo = mul128(block_reward, DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT, &hi); + div128_32(hi, lo, min_block_weight, &hi, &lo); + div128_32(hi, lo, median_block_weight, &hi, &lo); + assert(hi == 0); + lo /= 5; + return lo; + } - if (median_block_size < min_block_size) - median_block_size = min_block_size; + const uint64_t fee_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; - uint64_t unscaled_fee_per_kb = (fee_per_kb_base * min_block_size / median_block_size); - uint64_t hi, lo = mul128(unscaled_fee_per_kb, block_reward, &hi); + uint64_t unscaled_fee_base = (fee_base * min_block_weight / median_block_weight); + lo = mul128(unscaled_fee_base, block_reward, &hi); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD % 1000000 == 0, "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD must be divisible by 1000000"); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000 <= std::numeric_limits<uint32_t>::max(), "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD is too large"); @@ -3031,29 +2938,48 @@ uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median } //------------------------------------------------------------------ -bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const +bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); - uint64_t fee_per_kb; - if (version < HF_VERSION_DYNAMIC_FEE) - { - fee_per_kb = FEE_PER_KB; - } - else + uint64_t median = 0; + uint64_t already_generated_coins = 0; + uint64_t base_reward = 0; + if (version >= HF_VERSION_DYNAMIC_FEE) { - uint64_t median = m_current_block_cumul_sz_limit / 2; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - uint64_t base_reward; + median = m_current_block_cumul_weight_limit / 2; + already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; - fee_per_kb = get_dynamic_per_kb_fee(base_reward, median, version); } - MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); - uint64_t needed_fee = blob_size / 1024; - needed_fee += (blob_size % 1024) ? 1 : 0; - needed_fee *= fee_per_kb; + uint64_t needed_fee; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, median, version); + MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); + needed_fee = tx_weight * fee_per_byte; + // quantize fee up to 8 decimals + const uint64_t mask = get_fee_quantization_mask(); + needed_fee = (needed_fee + mask - 1) / mask * mask; + } + else + { + uint64_t fee_per_kb; + if (version < HF_VERSION_DYNAMIC_FEE) + { + fee_per_kb = FEE_PER_KB; + } + else + { + fee_per_kb = get_dynamic_base_fee(base_reward, median, version); + } + MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); + + needed_fee = tx_weight / 1024; + needed_fee += (tx_weight % 1024) ? 1 : 0; + needed_fee *= fee_per_kb; + } if (fee < needed_fee - needed_fee / 50) // keep a little 2% buffer on acceptance - no integer overflow { @@ -3064,7 +2990,7 @@ bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const +uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const { const uint8_t version = get_current_hard_fork_version(); @@ -3074,15 +3000,16 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons if (grace_blocks >= CRYPTONOTE_REWARD_BLOCKS_WINDOW) grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1; - const uint64_t min_block_size = get_min_block_size(version); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + const uint64_t min_block_weight = get_min_block_weight(version); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + weights.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) - sz.push_back(min_block_size); + weights.push_back(min_block_weight); - uint64_t median = epee::misc_utils::median(sz); - if(median <= min_block_size) - median = min_block_size; + uint64_t median = epee::misc_utils::median(weights); + if(median <= min_block_weight) + median = min_block_weight; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; uint64_t base_reward; @@ -3092,8 +3019,9 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_per_kb_fee(base_reward, median, version); - MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/kB"); + uint64_t fee = get_dynamic_base_fee(base_reward, median, version); + const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; + MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; } @@ -3234,6 +3162,7 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons // need most recent 60 blocks, get index of first of those size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; + timestamps.reserve(h - offset); for(;offset < h; ++offset) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -3260,7 +3189,7 @@ void Blockchain::return_tx_to_pool(std::vector<transaction> &txs) } } //------------------------------------------------------------------ -bool Blockchain::flush_txes_from_pool(const std::list<crypto::hash> &txids) +bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) { CRITICAL_REGION_LOCAL(m_tx_pool); @@ -3268,11 +3197,11 @@ bool Blockchain::flush_txes_from_pool(const std::list<crypto::hash> &txids) for (const auto &txid: txids) { cryptonote::transaction tx; - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -3298,6 +3227,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if(bl.prev_id != get_tail_id()) { MERROR_VER("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << get_tail_id()); + bvc.m_verifivation_failed = true; leave: m_db->block_txn_stop(); return false; @@ -3431,8 +3361,8 @@ leave: goto leave; } - size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); - size_t cumulative_block_size = coinbase_blob_size; + size_t coinbase_weight = get_transaction_weight(bl.miner_tx); + size_t cumulative_block_weight = coinbase_weight; std::vector<transaction> txs; key_images_container keys; @@ -3450,10 +3380,11 @@ leave: // Iterate over the block's transaction hashes, grabbing each // from the tx_pool and validating them. Each is then added // to txs. Keys spent in each are added to <keys> by the double spend check. + txs.reserve(bl.tx_hashes.size()); for (const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; - size_t blob_size = 0; + size_t tx_weight = 0; uint64_t fee = 0; bool relayed = false, do_not_relay = false, double_spend_seen = false; TIME_MEASURE_START(aa); @@ -3472,7 +3403,7 @@ leave: TIME_MEASURE_START(bb); // get transaction with hash <tx_id> from tx_pool - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(!m_tx_pool.take_tx(tx_id, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; @@ -3543,7 +3474,7 @@ leave: TIME_MEASURE_FINISH(cc); t_checktx += cc; fee_summary += fee; - cumulative_block_size += blob_size; + cumulative_block_weight += tx_weight; } m_blocks_txs_check.clear(); @@ -3551,7 +3482,7 @@ leave: TIME_MEASURE_START(vmt); uint64_t base_reward = 0; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) + if(!validate_miner_transaction(bl, cumulative_block_weight, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) { MERROR_VER("Block with id: " << id << " has incorrect miner transaction"); bvc.m_verifivation_failed = true; @@ -3560,11 +3491,11 @@ leave: } TIME_MEASURE_FINISH(vmt); - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; // populate various metadata about the block to be stored alongside it. - block_size = cumulative_block_size; + block_weight = cumulative_block_weight; cumulative_difficulty = current_diffic; // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins @@ -3585,7 +3516,7 @@ leave: { try { - new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + new_height = m_db->add_block(bl, block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { @@ -3598,6 +3529,7 @@ leave: { //TODO: figure out the best way to deal with this failure LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; } @@ -3609,14 +3541,14 @@ leave: TIME_MEASURE_FINISH(addblock); - // do this after updating the hard fork state since the size limit may change due to fork - update_next_cumulative_size_limit(); + // do this after updating the hard fork state since the weight limit may change due to fork + update_next_cumulative_weight_limit(); - MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); + MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); if(m_show_time_stats) { - MINFO("Height: " << new_height << " blob: " << coinbase_blob_size << " cumm: " - << cumulative_block_size << " p/t: " << block_processing_time << " (" + MINFO("Height: " << new_height << " coinbase weight: " << coinbase_weight << " cumm: " + << cumulative_block_weight << " p/t: " << block_processing_time << " (" << target_calculating_time << "/" << longhash_calculating_time << "/" << t1 << "/" << t2 << "/" << t3 << "/" << t_exists << "/" << t_pool << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock << ")ms"); @@ -3627,24 +3559,30 @@ leave: // appears to be a NOP *and* is called elsewhere. wat? m_tx_pool.on_blockchain_inc(new_height, id); + get_difficulty_for_next_block(); // just to cache it + invalidate_block_template_cache(); + + std::shared_ptr<tools::Notify> block_notify = m_block_notify; + if (block_notify) + block_notify->notify(epee::string_tools::pod_to_hex(id).c_str()); return true; } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_size_limit() +bool Blockchain::update_next_cumulative_weight_limit() { - uint64_t full_reward_zone = get_min_block_size(get_current_hard_fork_version()); + uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(sz); - m_current_block_cumul_sz_median = median; + uint64_t median = epee::misc_utils::median(weights); + m_current_block_cumul_weight_median = median; if(median <= full_reward_zone) median = full_reward_zone; - m_current_block_cumul_sz_limit = median*2; + m_current_block_cumul_weight_limit = median*2; return true; } //------------------------------------------------------------------ @@ -3807,11 +3745,13 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) store_blockchain(); m_sync_counter = 0; } - else if (m_db_blocks_per_sync && m_sync_counter >= m_db_blocks_per_sync) + else if (m_db_sync_threshold && ((m_db_sync_on_blocks && m_sync_counter >= m_db_sync_threshold) || (!m_db_sync_on_blocks && m_bytes_to_sync >= m_db_sync_threshold))) { + MDEBUG("Sync threshold met, syncing"); if(m_db_sync_mode == db_async) { m_sync_counter = 0; + m_bytes_to_sync = 0; m_async_service.dispatch(boost::bind(&Blockchain::store_blockchain, this)); } else if(m_db_sync_mode == db_sync) @@ -3863,7 +3803,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin } } -uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) +uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { // new: . . . . . X X X X X . . . . . . // pre: A A A A B B B B C C C C D D D D @@ -3966,12 +3906,13 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list<c // vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries // and is threaded if possible. The table (m_scan_table) will be used later when querying output // keys. -bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks_entry) +bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry) { MTRACE("Blockchain::" << __func__); TIME_MEASURE_START(prepare); bool stop_batch; uint64_t bytes = 0; + size_t total_txs = 0; // Order of locking must be: // m_incoming_tx_lock (optional) @@ -4000,7 +3941,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e { bytes += tx_blob.size(); } + total_txs += entry.txs.size(); } + m_bytes_to_sync += bytes; while (!(stop_batch = m_db->batch_start(blocks_entry.size(), bytes))) { m_blockchain_lock.unlock(); m_tx_pool.unlock(); @@ -4032,6 +3975,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (uint64_t i = 0; i < threads; i++) { + blocks[i].reserve(batches + 1); for (int j = 0; j < batches; j++) { block block; @@ -4058,7 +4002,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e break; } - blocks[i].push_back(block); + blocks[i].push_back(std::move(block)); std::advance(it, 1); } } @@ -4079,7 +4023,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e break; } - blocks[i].push_back(block); + blocks[i].push_back(std::move(block)); std::advance(it, 1); } @@ -4090,11 +4034,11 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e tools::threadpool::waiter waiter; for (uint64_t i = 0; i < threads; i++) { - tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i]))); + tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i])), true); thread_height += blocks[i].size(); } - waiter.wait(); + waiter.wait(&tpool); if (m_cancel) return false; @@ -4135,6 +4079,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e std::map<uint64_t, std::vector<uint64_t>> offset_map; // [output] stores all output_data_t for each absolute_offset std::map<uint64_t, std::vector<output_data_t>> tx_map; + std::vector<std::pair<cryptonote::transaction, crypto::hash>> txes(total_txs); #define SCAN_TABLE_QUIT(m) \ do { \ @@ -4144,6 +4089,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e } while(0); \ // generate sorted tables for all amounts and absolute offsets + size_t tx_index = 0; for (const auto &entry : blocks_entry) { if (m_cancel) @@ -4151,12 +4097,15 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (const auto &tx_blob : entry.txs) { - crypto::hash tx_hash = null_hash; - crypto::hash tx_prefix_hash = null_hash; - transaction tx; + if (tx_index >= txes.size()) + SCAN_TABLE_QUIT("tx_index is out of sync"); + transaction &tx = txes[tx_index].first; + crypto::hash &tx_prefix_hash = txes[tx_index].second; + ++tx_index; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) + if (!parse_and_validate_tx_base_from_blob(tx_blob, tx)) SCAN_TABLE_QUIT("Could not parse tx from incoming blocks."); + cryptonote::get_transaction_prefix_hash(tx, tx_prefix_hash); auto its = m_scan_table.find(tx_prefix_hash); if (its != m_scan_table.end()) @@ -4229,9 +4178,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i]))); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i])), true); } - waiter.wait(); + waiter.wait(&tpool); } else { @@ -4242,9 +4191,8 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e } } - int total_txs = 0; - // now generate a table for each tx_prefix and k_image hashes + tx_index = 0; for (const auto &entry : blocks_entry) { if (m_cancel) @@ -4252,14 +4200,12 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (const auto &tx_blob : entry.txs) { - crypto::hash tx_hash = null_hash; - crypto::hash tx_prefix_hash = null_hash; - transaction tx; - - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) - SCAN_TABLE_QUIT("Could not parse tx from incoming blocks."); + if (tx_index >= txes.size()) + SCAN_TABLE_QUIT("tx_index is out of sync"); + const transaction &tx = txes[tx_index].first; + const crypto::hash &tx_prefix_hash = txes[tx_index].second; + ++tx_index; - ++total_txs; auto its = m_scan_table.find(tx_prefix_hash); if (its == m_scan_table.end()) SCAN_TABLE_QUIT("Tx not found on scan table from incoming blocks."); @@ -4348,7 +4294,7 @@ bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, con return m_db->for_all_txpool_txes(f, include_blob, include_unrelayed_txes); } -void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync) +void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync) { if (sync_mode == db_defaultsync) { @@ -4357,7 +4303,8 @@ void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, } m_db_sync_mode = sync_mode; m_fast_sync = fast_sync; - m_db_blocks_per_sync = blocks_per_sync; + m_db_sync_on_blocks = sync_on_blocks; + m_db_sync_threshold = sync_threshold; m_max_prepare_blocks_threads = maxthreads; } @@ -4378,6 +4325,39 @@ HardFork::State Blockchain::get_hard_fork_state() const return m_hardfork->get_state(); } +const std::vector<HardFork::Params>& Blockchain::get_hard_fork_heights(network_type nettype) +{ + static const std::vector<HardFork::Params> mainnet_heights = []() + { + std::vector<HardFork::Params> heights; + for (const auto& i : mainnet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector<HardFork::Params> testnet_heights = []() + { + std::vector<HardFork::Params> heights; + for (const auto& i : testnet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector<HardFork::Params> stagenet_heights = []() + { + std::vector<HardFork::Params> heights; + for (const auto& i : stagenet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector<HardFork::Params> dummy; + switch (nettype) + { + case MAINNET: return mainnet_heights; + case TESTNET: return testnet_heights; + case STAGENET: return stagenet_heights; + default: return dummy; + } +} + bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, uint32_t &votes, uint32_t &threshold, uint64_t &earliest_height, uint8_t &voting) const { return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting); @@ -4393,9 +4373,9 @@ std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> Blockchain:: get_ou return m_db->get_output_histogram(amounts, unlocked, recent_cutoff, min_count); } -std::list<std::pair<Blockchain::block_extended_info,uint64_t>> Blockchain::get_alternative_chains() const +std::list<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> Blockchain::get_alternative_chains() const { - std::list<std::pair<Blockchain::block_extended_info,uint64_t>> chains; + std::list<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> chains; for (const auto &i: m_alternative_chains) { @@ -4411,15 +4391,16 @@ std::list<std::pair<Blockchain::block_extended_info,uint64_t>> Blockchain::get_a } if (!found) { - uint64_t length = 1; + std::vector<crypto::hash> chain; auto h = i.second.bl.prev_id; + chain.push_back(top); blocks_ext_by_hash::const_iterator prev; while ((prev = m_alternative_chains.find(h)) != m_alternative_chains.end()) { + chain.push_back(h); h = prev->second.bl.prev_id; - ++length; } - chains.push_back(std::make_pair(i.second, length)); + chains.push_back(std::make_pair(i.second, chain)); } } return chains; @@ -4431,20 +4412,22 @@ void Blockchain::cancel() } #if defined(PER_BLOCK_CHECKPOINT) -static const char expected_block_hashes_hash[] = "0924bc1c47aae448321fde949554be192878dd800e6489379865218f84eacbca"; -void Blockchain::load_compiled_in_block_hashes() +static const char expected_block_hashes_hash[] = "954cb2bbfa2fe6f74b2cdd22a1a4c767aea249ad47ad4f7c9445f0f03260f511"; +void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints) { - const bool testnet = m_nettype == TESTNET; - const bool stagenet = m_nettype == STAGENET; - if (m_fast_sync && get_blocks_dat_start(testnet, stagenet) != nullptr && get_blocks_dat_size(testnet, stagenet) > 0) + if (get_checkpoints == nullptr || !m_fast_sync) { - MINFO("Loading precomputed blocks (" << get_blocks_dat_size(testnet, stagenet) << " bytes)"); - + return; + } + const epee::span<const unsigned char> &checkpoints = get_checkpoints(m_nettype); + if (!checkpoints.empty()) + { + MINFO("Loading precomputed blocks (" << checkpoints.size() << " bytes)"); if (m_nettype == MAINNET) { // first check hash crypto::hash hash; - if (!tools::sha256sum(get_blocks_dat_start(testnet, stagenet), get_blocks_dat_size(testnet, stagenet), hash)) + if (!tools::sha256sum(checkpoints.data(), checkpoints.size(), hash)) { MERROR("Failed to hash precomputed blocks data"); return; @@ -4464,9 +4447,9 @@ void Blockchain::load_compiled_in_block_hashes() } } - if (get_blocks_dat_size(testnet, stagenet) > 4) + if (checkpoints.size() > 4) { - const unsigned char *p = get_blocks_dat_start(testnet, stagenet); + const unsigned char *p = checkpoints.data(); const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24); if (nblocks > (std::numeric_limits<uint32_t>::max() - 4) / sizeof(hash)) { @@ -4474,7 +4457,7 @@ void Blockchain::load_compiled_in_block_hashes() return; } const size_t size_needed = 4 + nblocks * sizeof(crypto::hash); - if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && get_blocks_dat_size(testnet, stagenet) >= size_needed) + if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && checkpoints.size() >= size_needed) { p += sizeof(uint32_t); m_blocks_hash_of_hashes.reserve(nblocks); @@ -4495,17 +4478,17 @@ void Blockchain::load_compiled_in_block_hashes() // for tx hashes will fail in handle_block_to_main_chain(..) CRITICAL_REGION_LOCAL(m_tx_pool); - std::list<transaction> txs; + std::vector<transaction> txs; m_tx_pool.get_transactions(txs); - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; transaction pool_tx; for(const transaction &tx : txs) { crypto::hash tx_hash = get_transaction_hash(tx); - m_tx_pool.take_tx(tx_hash, pool_tx, blob_size, fee, relayed, do_not_relay, double_spend_seen); + m_tx_pool.take_tx(tx_hash, pool_tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen); } } } @@ -4557,7 +4540,25 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t he return m_db->for_all_outputs(amount, f);; } +void Blockchain::invalidate_block_template_cache() +{ + MDEBUG("Invalidating block template cache"); + m_btc_valid = false; +} + +void Blockchain::cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t expected_reward, uint64_t pool_cookie) +{ + MDEBUG("Setting block template cache"); + m_btc = b; + m_btc_address = address; + m_btc_nonce = nonce; + m_btc_difficulty = diff; + m_btc_expected_reward = expected_reward; + m_btc_pool_cookie = pool_cookie; + m_btc_valid = true; +} + namespace cryptonote { -template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::list<transaction>&, std::list<crypto::hash>&) const; -template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::list<cryptonote::blobdata>&, std::list<crypto::hash>&, bool) const; +template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; +template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index ef736d1e7..f140d7719 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -38,9 +38,11 @@ #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/member.hpp> #include <atomic> +#include <functional> #include <unordered_map> #include <unordered_set> +#include "span.h" #include "syncobj.h" #include "string_tools.h" #include "cryptonote_basic/cryptonote_basic.h" @@ -55,6 +57,8 @@ #include "cryptonote_basic/hardfork.h" #include "blockchain_db/blockchain_db.h" +namespace tools { class Notify; } + namespace cryptonote { class tx_memory_pool; @@ -71,6 +75,15 @@ namespace cryptonote db_nosync //!< Leave syncing up to the backing db (safest, but slowest because of disk I/O) }; + /** + * @brief Callback routine that returns checkpoints data for specific network type + * + * @param network network type + * + * @return checkpoints data, empty span if there ain't any checkpoints for specific network type + */ + typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback; + /************************************************************************/ /* */ /************************************************************************/ @@ -95,7 +108,7 @@ namespace cryptonote { block bl; //!< the block uint64_t height; //!< the height of the block in the blockchain - size_t block_cumulative_size; //!< the size (in bytes) of the block + size_t block_cumulative_weight; //!< the weight of the block difficulty_type cumulative_difficulty; //!< the accumulated difficulty after that block uint64_t already_generated_coins; //!< the total coins minted after that block }; @@ -114,10 +127,12 @@ namespace cryptonote * @param nettype network type * @param offline true if running offline, else false * @param test_options test parameters + * @param fixed_difficulty fixed difficulty for testing purposes; 0 means disabled + * @param get_checkpoints if set, will be called to get checkpoints data * * @return true on success, false if any initialization steps fail */ - bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL); + bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL, difficulty_type fixed_difficulty = 0, const GetCheckpointsCallback& get_checkpoints = nullptr); /** * @brief Initialize the Blockchain state @@ -157,7 +172,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const; /** * @brief get blocks from blocks based on start height and count @@ -168,7 +183,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const; /** * @brief compiles a list of all blocks stored as alternative chains @@ -177,7 +192,7 @@ namespace cryptonote * * @return true */ - bool get_alternative_blocks(std::list<block>& blocks) const; + bool get_alternative_blocks(std::vector<block>& blocks) const; /** * @brief returns the number of alternative blocks stored @@ -213,7 +228,7 @@ namespace cryptonote * * @return false on erroneous blocks, else true */ - bool prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); /** * @brief incoming blocks post-processing, cleanup, and disk sync @@ -373,7 +388,7 @@ namespace cryptonote * * @return true if a block found in common, else false */ - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::list<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const; /** * @brief get recent block hashes for a foreign chain @@ -420,7 +435,7 @@ namespace cryptonote * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions @@ -446,16 +461,6 @@ namespace cryptonote uint64_t get_num_mature_outputs(uint64_t amount) const; /** - * @brief get random outputs (indices) for an amount - * - * @param amount the amount - * @param count the number of random outputs to choose - * - * @return the outputs' amount-global indices - */ - std::vector<uint64_t> get_random_outputs(uint64_t amount, uint64_t count) const; - - /** * @brief get the public key for an output * * @param amount the output amount @@ -466,22 +471,6 @@ namespace cryptonote crypto::public_key get_output_key(uint64_t amount, uint64_t global_index) const; /** - * @brief gets random outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @brief gets specific outputs to mix with * * This function takes an RPC request for outputs to mix with @@ -508,23 +497,6 @@ namespace cryptonote void get_output_key_mask_unlocked(const uint64_t& amount, const uint64_t& index, crypto::public_key& key, rct::key& mask, bool& unlocked) const; /** - * @brief gets random ringct outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices - * and the matching keys. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @brief gets per block distribution of outputs of a given amount * * @param amount the amount to get a ditribution for @@ -578,46 +550,57 @@ namespace cryptonote bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false); /** - * @brief get dynamic per kB fee for a given block size + * @brief get fee quantization mask * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. + * The dynamic fee may be quantized, to mask out the last decimal places + * + * @return the fee quantized mask + */ + static uint64_t get_fee_quantization_mask(); + + /** + * @brief get dynamic per kB or byte fee for a given block weight + * + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. * * @param block_reward the current block reward - * @param median_block_size the median blob's size in the past window + * @param median_block_weight the median block weight in the past window * @param version hard fork version for rules and constants to use * - * @return the per kB fee + * @return the fee */ - static uint64_t get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version); + static uint64_t get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version); /** - * @brief get dynamic per kB fee estimate for the next few blocks + * @brief get dynamic per kB or byte fee estimate for the next few blocks * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. This function - * calculates an estimate for a dynamic fee which will be valid for - * the next grace_blocks + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. + * This function calculates an estimate for a dynamic fee which will be + * valid for the next grace_blocks * * @param grace_blocks number of blocks we want the fee to be valid for * - * @return the per kB fee estimate + * @return the fee estimate */ - uint64_t get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const; + uint64_t get_dynamic_base_fee_estimate(uint64_t grace_blocks) const; /** * @brief validate a transaction's fee * * This function validates the fee is enough for the transaction. - * This is based on the size of the transaction blob, and, after a - * height threshold, on the average size of transaction in a past window + * This is based on the weight of the transaction, and, after a + * height threshold, on the average weight of transaction in a past window * - * @param blob_size the transaction blob's size + * @param tx_weight the transaction weight * @param fee the fee * * @return true if the fee is enough, false otherwise */ - bool check_fee(size_t blob_size, uint64_t fee) const; + bool check_fee(size_t tx_weight, uint64_t fee) const; /** * @brief check that a transaction's outputs conform to current standards @@ -634,18 +617,18 @@ namespace cryptonote bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc); /** - * @brief gets the blocksize limit based on recent blocks + * @brief gets the block weight limit based on recent blocks * * @return the limit */ - uint64_t get_current_cumulative_blocksize_limit() const; + uint64_t get_current_cumulative_block_weight_limit() const; /** - * @brief gets the blocksize median based on recent blocks (same window as for the limit) + * @brief gets the block weight median based on recent blocks (same window as for the limit) * * @return the median */ - uint64_t get_current_cumulative_blocksize_median() const; + uint64_t get_current_cumulative_block_weight_median() const; /** * @brief gets the difficulty of the block with a given height @@ -728,14 +711,22 @@ namespace cryptonote * @brief sets various performance options * * @param maxthreads max number of threads when preparing blocks for addition - * @param blocks_per_sync number of blocks to cache before syncing to database + * @param sync_on_blocks whether to sync based on blocks or bytes + * @param sync_threshold number of blocks/bytes to cache before syncing to database * @param sync_mode the ::blockchain_db_sync_mode to use * @param fast_sync sync using built-in block hashes as trusted */ - void set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, + void set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync); /** + * @brief sets a block notify object to call for every new block + * + * @param notify the notify object to cal at every new block + */ + void set_block_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_block_notify = notify; } + + /** * @brief Put DB in safe sync mode */ void safesyncmode(const bool onoff); @@ -755,6 +746,13 @@ namespace cryptonote HardFork::State get_hard_fork_state() const; /** + * @brief gets the hardfork heights of given network + * + * @return the HardFork object + */ + static const std::vector<HardFork::Params>& get_hard_fork_heights(network_type nettype); + + /** * @brief gets the current hardfork version in use/voted for * * @return the version @@ -829,7 +827,7 @@ namespace cryptonote * * @return false if any removals fail, otherwise true */ - bool flush_txes_from_pool(const std::list<crypto::hash> &txids); + bool flush_txes_from_pool(const std::vector<crypto::hash> &txids); /** * @brief return a histogram of outputs on the blockchain @@ -939,7 +937,7 @@ namespace cryptonote * * @return a list of chains */ - std::list<std::pair<block_extended_info,uint64_t>> get_alternative_chains() const; + std::list<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const; void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta); void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); @@ -952,7 +950,7 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); void lock(); void unlock(); @@ -966,6 +964,11 @@ namespace cryptonote */ void on_new_tx_from_block(const cryptonote::transaction &tx); + /** + * @brief returns the timestamps of the last N blocks + */ + std::vector<time_t> get_last_block_timestamps(unsigned int blocks) const; + private: // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage @@ -992,8 +995,8 @@ namespace cryptonote // main chain transactions_container m_transactions; - size_t m_current_block_cumul_sz_limit; - size_t m_current_block_cumul_sz_median; + size_t m_current_block_cumul_weight_limit; + size_t m_current_block_cumul_weight_median; // metadata containers std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table; @@ -1009,11 +1012,13 @@ namespace cryptonote bool m_fast_sync; bool m_show_time_stats; bool m_db_default_sync; - uint64_t m_db_blocks_per_sync; + bool m_db_sync_on_blocks; + uint64_t m_db_sync_threshold; uint64_t m_max_prepare_blocks_threads; uint64_t m_fake_pow_calc_time; uint64_t m_fake_scan_time; uint64_t m_sync_counter; + uint64_t m_bytes_to_sync; std::vector<uint64_t> m_timestamps; std::vector<difficulty_type> m_difficulties; uint64_t m_timestamps_and_difficulties_height; @@ -1040,9 +1045,21 @@ namespace cryptonote network_type m_nettype; bool m_offline; + difficulty_type m_fixed_difficulty; std::atomic<bool> m_cancel; + // block template cache + block m_btc; + account_public_address m_btc_address; + blobdata m_btc_nonce; + difficulty_type m_btc_difficulty; + uint64_t m_btc_pool_cookie; + uint64_t m_btc_expected_reward; + bool m_btc_valid; + + std::shared_ptr<tools::Notify> m_block_notify; + /** * @brief collects the keys for all outputs being "spent" as an input * @@ -1204,7 +1221,7 @@ namespace cryptonote * and that his miner transaction totals reward + fee. * * @param b the block containing the miner transaction to be validated - * @param cumulative_block_size the block's size + * @param cumulative_block_weight the block's weight * @param fee the total fees collected in the block * @param base_reward return-by-reference the new block's generated coins * @param already_generated_coins the amount of currency generated prior to this block @@ -1213,7 +1230,7 @@ namespace cryptonote * * @return false if anything is found wrong with the miner transaction, otherwise true */ - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); + bool validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); /** * @brief reverts the blockchain to its previous state following a failed switch @@ -1230,32 +1247,14 @@ namespace cryptonote bool rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height); /** - * @brief gets recent block sizes for median calculation + * @brief gets recent block weights for median calculation * - * get the block sizes of the last <count> blocks, and return by reference <sz>. + * get the block weights of the last <count> blocks, and return by reference <sz>. * - * @param sz return-by-reference the list of sizes - * @param count the number of blocks to get sizes for + * @param sz return-by-reference the list of weights + * @param count the number of blocks to get weights for */ - void get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; - - /** - * @brief adds the given output to the requested set of random outputs - * - * @param result_outs return-by-reference the set the output is to be added to - * @param amount the output amount - * @param i the output index (indexed to amount) - */ - void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; - - /** - * @brief adds the given output to the requested set of random ringct outputs - * - * @param outs return-by-reference the set the output is to be added to - * @param amount the output amount (0 for rct inputs) - * @param i the rct output index - */ - void add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const; + void get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) @@ -1352,11 +1351,11 @@ namespace cryptonote bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); /** - * @brief calculate the block size limit for the next block to be added + * @brief calculate the block weight limit for the next block to be added * * @return true */ - bool update_next_cumulative_size_limit(); + bool update_next_cumulative_weight_limit(); void return_tx_to_pool(std::vector<transaction> &txs); /** @@ -1387,8 +1386,10 @@ namespace cryptonote * A (possibly empty) set of block hashes can be compiled into the * monero daemon binary. This function loads those hashes into * a useful state. + * + * @param get_checkpoints if set, will be called to get checkpoints data */ - void load_compiled_in_block_hashes(); + void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints); /** * @brief expands v2 transaction data from blockchain @@ -1398,5 +1399,17 @@ namespace cryptonote * that implicit data. */ bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys); + + /** + * @brief invalidates any cached block template + */ + void invalidate_block_template_cache(); + + /** + * @brief stores a new cached block template + * + * At some point, may be used to push an update to miners + */ + void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t expected_reward, uint64_t pool_cookie); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 7d34415c4..4b806c282 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -53,6 +53,7 @@ using namespace epee; #include "ringct/rctTypes.h" #include "blockchain_db/blockchain_db.h" #include "ringct/rctSigs.h" +#include "common/notify.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -76,6 +77,16 @@ namespace cryptonote , "Run on stagenet. The wallet must be launched with --stagenet flag." , false }; + const command_line::arg_descriptor<bool> arg_regtest_on = { + "regtest" + , "Run in a regression testing mode." + , false + }; + const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty = { + "fixed-difficulty" + , "Fixed difficulty used for testing." + , 0 + }; const command_line::arg_descriptor<std::string, false, true, 2> arg_data_dir = { "data-dir" , "Specify data directory" @@ -152,10 +163,15 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; - static const command_line::arg_descriptor<size_t> arg_max_txpool_size = { - "max-txpool-size" - , "Set maximum txpool size in bytes." - , DEFAULT_TXPOOL_MAX_SIZE + static const command_line::arg_descriptor<size_t> arg_max_txpool_weight = { + "max-txpool-weight" + , "Set maximum txpool weight in bytes." + , DEFAULT_TXPOOL_MAX_WEIGHT + }; + static const command_line::arg_descriptor<std::string> arg_block_notify = { + "block-notify" + , "Run a program for each new block, '%s' will be replaced by the block hash" + , "" }; //----------------------------------------------------------------------------------------------- @@ -170,9 +186,9 @@ namespace cryptonote m_last_dns_checkpoints_update(0), m_last_json_checkpoints_update(0), m_disable_dns_checkpoints(false), - m_threadpool(tools::threadpool::getInstance()), m_update_download(0), - m_nettype(UNDEFINED) + m_nettype(UNDEFINED), + m_update_available(false) { m_checkpoints_updating.clear(); set_cryptonote_protocol(pprotocol); @@ -252,6 +268,8 @@ namespace cryptonote command_line::add_arg(desc, arg_testnet_on); command_line::add_arg(desc, arg_stagenet_on); + command_line::add_arg(desc, arg_regtest_on); + command_line::add_arg(desc, arg_fixed_difficulty); command_line::add_arg(desc, arg_dns_checkpoints); command_line::add_arg(desc, arg_prep_blocks_threads); command_line::add_arg(desc, arg_fast_block_sync); @@ -263,7 +281,8 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); - command_line::add_arg(desc, arg_max_txpool_size); + command_line::add_arg(desc, arg_max_txpool_weight); + command_line::add_arg(desc, arg_block_notify); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -324,19 +343,19 @@ namespace cryptonote top_id = m_blockchain_storage.get_tail_id(height); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks, txs); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<block>& blocks) const { - std::list<std::pair<cryptonote::blobdata, cryptonote::block>> bs; + std::vector<std::pair<cryptonote::blobdata, cryptonote::block>> bs; if (!m_blockchain_storage.get_blocks(start_offset, count, bs)) return false; for (const auto &b: bs) @@ -344,7 +363,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::blobdata>& txs, std::list<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } @@ -355,12 +374,12 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- - bool core::get_alternative_blocks(std::list<block>& blocks) const + bool core::get_alternative_blocks(std::vector<block>& blocks) const { return m_blockchain_storage.get_alternative_blocks(blocks); } @@ -370,11 +389,12 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, const char *config_subdir, const cryptonote::test_options *test_options) + bool core::init(const boost::program_options::variables_map& vm, const char *config_subdir, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */) { start_time = std::time(nullptr); - if (test_options != NULL) + const bool regtest = command_line::get_arg(vm, arg_regtest_on); + if (test_options != NULL || regtest) { m_nettype = FAKECHAIN; } @@ -390,7 +410,7 @@ namespace cryptonote bool fast_sync = command_line::get_arg(vm, arg_fast_block_sync) != 0; uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); - size_t max_txpool_size = command_line::get_arg(vm, arg_max_txpool_size); + size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -427,9 +447,20 @@ namespace cryptonote MGINFO("Loading blockchain from folder " << folder.string() << " ..."); const std::string filename = folder.string(); - // default to fast:async:1 + // default to fast:async:1 if overridden blockchain_db_sync_mode sync_mode = db_defaultsync; - uint64_t blocks_per_sync = 1; + bool sync_on_blocks = true; + uint64_t sync_threshold = 1; + + if (m_nettype == FAKECHAIN) + { + // reset the db by removing the database file before opening it + if (!db->remove_data_file(filename)) + { + MERROR("Failed to remove data file in " << filename); + return false; + } + } try { @@ -469,7 +500,7 @@ namespace cryptonote else if(options[0] == "fastest") { db_flags = DBF_FASTEST; - blocks_per_sync = 1000; // default to fastest:async:1000 + sync_threshold = 1000; // default to fastest:async:1000 sync_mode = db_sync_mode_is_default ? db_defaultsync : db_async; } else @@ -487,9 +518,22 @@ namespace cryptonote if(options.size() >= 3 && !safemode) { char *endptr; - uint64_t bps = strtoull(options[2].c_str(), &endptr, 0); - if (*endptr == '\0') - blocks_per_sync = bps; + uint64_t threshold = strtoull(options[2].c_str(), &endptr, 0); + if (*endptr == '\0' || !strcmp(endptr, "blocks")) + { + sync_on_blocks = true; + sync_threshold = threshold; + } + else if (!strcmp(endptr, "bytes")) + { + sync_on_blocks = false; + sync_threshold = threshold; + } + else + { + LOG_ERROR("Invalid db sync mode: " << options[2]); + return false; + } } if (db_salvage) @@ -506,11 +550,26 @@ namespace cryptonote } m_blockchain_storage.set_user_options(blocks_threads, - blocks_per_sync, sync_mode, fast_sync); + sync_on_blocks, sync_threshold, sync_mode, fast_sync); + + try + { + if (!command_line::is_arg_defaulted(vm, arg_block_notify)) + m_blockchain_storage.set_block_notify(std::shared_ptr<tools::Notify>(new tools::Notify(command_line::get_arg(vm, arg_block_notify).c_str()))); + } + catch (const std::exception &e) + { + MERROR("Failed to parse block notify spec"); + } - r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, test_options); + const std::pair<uint8_t, uint64_t> regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; + const cryptonote::test_options regtest_test_options = { + regtest_hard_forks + }; + const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); + r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty, get_checkpoints); - r = m_mempool.init(max_txpool_size); + r = m_mempool.init(max_txpool_weight); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); // now that we have a valid m_blockchain_storage, we can clean out any @@ -651,40 +710,139 @@ namespace cryptonote return false; } + return true; + } + //----------------------------------------------------------------------------------------------- + void core::set_semantics_failed(const crypto::hash &tx_hash) + { + LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + bad_semantics_txes_lock.lock(); + bad_semantics_txes[0].insert(tx_hash); + if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + { + std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); + bad_semantics_txes[0].clear(); + } + bad_semantics_txes_lock.unlock(); + } + //----------------------------------------------------------------------------------------------- + static bool is_canonical_bulletproof_layout(const std::vector<rct::Bulletproof> &proofs) + { + if (proofs.size() != 1) + return false; + const size_t sz = proofs[0].V.size(); + if (sz == 0 || sz > BULLETPROOF_MAX_OUTPUTS) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block) + { + bool ret = true; if (keeped_by_block && get_blockchain_storage().is_within_compiled_block_hash_area()) { MTRACE("Skipping semantics check for tx kept by block in embedded hash area"); + return true; } - else if(!check_tx_semantic(tx, keeped_by_block)) + + std::vector<const rct::rctSig*> rvv; + for (size_t n = 0; n < tx_info.size(); ++n) { - LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); - tvc.m_verifivation_failed = true; - bad_semantics_txes_lock.lock(); - bad_semantics_txes[0].insert(tx_hash); - if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + if (!check_tx_semantic(*tx_info[n].tx, keeped_by_block)) { - std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); - bad_semantics_txes[0].clear(); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + continue; + } + + if (tx_info[n].tx->version < 2) + continue; + const rct::rctSig &rv = tx_info[n].tx->rct_signatures; + switch (rv.type) { + case rct::RCTTypeNull: + // coinbase should not come here, so we reject for all other types + MERROR_VER("Unexpected Null rctSig type"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + case rct::RCTTypeSimple: + if (!rct::verRctSemanticsSimple(rv)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeFull: + if (!rct::verRct(rv, true)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeBulletproof: + if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) + { + MERROR_VER("Bulletproof does not have canonical form"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + rvv.push_back(&rv); // delayed batch verification + break; + default: + MERROR_VER("Unknown rct type: " << rv.type); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + } + if (!rvv.empty() && !rct::verRctSemanticsSimple(rvv)) + { + LOG_PRINT_L1("One transaction among this group has bad semantics, verifying one at a time"); + ret = false; + const bool assumed_bad = rvv.size() == 1; // if there's only one tx, it must be the bad one + for (size_t n = 0; n < tx_info.size(); ++n) + { + if (!tx_info[n].result) + continue; + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof) + continue; + if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) + { + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + } } - bad_semantics_txes_lock.unlock(); - return false; } - return true; + return ret; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { TRY_ENTRY(); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; bool in_txpool; bool in_blockchain; }; std::vector<result> results(tx_blobs.size()); tvc.resize(tx_blobs.size()); + tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - std::list<blobdata>::const_iterator it = tx_blobs.begin(); + std::vector<blobdata>::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); @@ -696,22 +854,25 @@ namespace cryptonote } }); } - waiter.wait(); + waiter.wait(&tpool); it = tx_blobs.begin(); + std::vector<bool> already_have(tx_blobs.size(), false); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { if (!results[i].res) continue; if(m_mempool.have_tx(results[i].hash)) { LOG_PRINT_L2("tx " << results[i].hash << "already have transaction in tx_pool"); + already_have[i] = true; } else if(m_blockchain_storage.have_tx(results[i].hash)) { LOG_PRINT_L2("tx " << results[i].hash << " already have transaction in blockchain"); + already_have[i] = true; } else { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); @@ -724,18 +885,31 @@ namespace cryptonote }); } } - waiter.wait(); + waiter.wait(&tpool); + + std::vector<tx_verification_batch_info> tx_info; + tx_info.reserve(tx_blobs.size()); + for (size_t i = 0; i < tx_blobs.size(); i++) { + if (!results[i].res || already_have[i]) + continue; + tx_info.push_back({&results[i].tx, results[i].hash, tvc[i], results[i].res}); + } + if (!tx_info.empty()) + handle_incoming_tx_accumulated_batch(tx_info, keeped_by_block); bool ok = true; it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { + if (already_have[i]) + continue; if (!results[i].res) { ok = false; continue; } - ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, it->size(), tvc[i], keeped_by_block, relayed, do_not_relay); + const size_t weight = get_transaction_weight(results[i].tx, it->size()); + ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -751,7 +925,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - std::list<cryptonote::blobdata> tx_blobs; + std::vector<cryptonote::blobdata> tx_blobs; tx_blobs.push_back(tx_blob); std::vector<tx_verification_context> tvcv(1); bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay); @@ -818,9 +992,9 @@ namespace cryptonote } // for version > 1, ringct signatures check verifies amounts match - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_transaction_weight(tx) >= m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - MERROR_VER("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + MERROR_VER("tx is too large " << get_transaction_weight(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -843,36 +1017,6 @@ namespace cryptonote return false; } - if (tx.version >= 2) - { - const rct::rctSig &rv = tx.rct_signatures; - switch (rv.type) { - case rct::RCTTypeNull: - // coinbase should not come here, so we reject for all other types - MERROR_VER("Unexpected Null rctSig type"); - return false; - case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: - if (!rct::verRctSimple(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: - if (!rct::verRct(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - default: - MERROR_VER("Unknown rct type: " << rv.type); - return false; - } - } - return true; } //----------------------------------------------------------------------------------------------- @@ -917,8 +1061,8 @@ namespace cryptonote const uint64_t end = start_offset + count - 1; m_blockchain_storage.for_blocks_range(start_offset, end, [this, &emission_amount, &total_fee_amount](uint64_t, const crypto::hash& hash, const block& b){ - std::list<transaction> txs; - std::list<crypto::hash> missed_txs; + std::vector<transaction> txs; + std::vector<crypto::hash> missed_txs; uint64_t coinbase_amount = get_outs_money_amount(b.miner_tx); this->get_transactions(b.tx_hashes, txs, missed_txs); uint64_t tx_fee_amount = 0; @@ -982,7 +1126,8 @@ namespace cryptonote crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); blobdata bl; t_serializable_object_to_blob(tx, bl); - return add_new_tx(tx, tx_hash, tx_prefix_hash, bl.size(), tvc, keeped_by_block, relayed, do_not_relay); + size_t tx_weight = get_transaction_weight(tx, bl.size()); + return add_new_tx(tx, tx_hash, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -990,7 +1135,7 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if (keeped_by_block) get_blockchain_storage().on_new_tx_from_block(tx); @@ -1008,13 +1153,13 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() { // we attempt to relay txes that should be relayed, but were not - std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs; + std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; if (m_mempool.get_relayable_transactions(txs) && !txs.empty()) { cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); @@ -1032,7 +1177,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob) { - std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs; + std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; cryptonote::transaction tx; crypto::hash tx_hash, tx_prefix_hash; if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) @@ -1054,14 +1199,9 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { - return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, max_count); - } - //----------------------------------------------------------------------------------------------- - bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const - { - return m_blockchain_storage.get_random_outs_for_amounts(req, res); + return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const @@ -1069,11 +1209,6 @@ namespace cryptonote return m_blockchain_storage.get_outs(req, res); } //----------------------------------------------------------------------------------------------- - bool core::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const - { - return m_blockchain_storage.get_random_rct_outs(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const { return m_blockchain_storage.get_output_distribution(amount, from_height, to_height, start_height, distribution, base); @@ -1111,7 +1246,7 @@ namespace cryptonote { block_verification_context bvc = boost::value_initialized<block_verification_context>(); m_miner.pause(); - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; try { blocks.push_back(get_block_complete_entry(b, m_mempool)); @@ -1135,8 +1270,8 @@ namespace cryptonote cryptonote_connection_context exclude_context = boost::value_initialized<cryptonote_connection_context>(); NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); - std::list<crypto::hash> missed_txs; - std::list<cryptonote::blobdata> txs; + std::vector<crypto::hash> missed_txs; + std::vector<cryptonote::blobdata> txs; m_blockchain_storage.get_transactions_blobs(b.tx_hashes, txs, missed_txs); if(missed_txs.size() && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) { @@ -1172,7 +1307,7 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks) + bool core::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks) { m_incoming_tx_lock.lock(); m_blockchain_storage.prepare_handle_incoming_blocks(blocks); @@ -1265,7 +1400,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_transactions(std::list<transaction>& txs, bool include_sensitive_data) const + bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); return true; @@ -1359,6 +1494,7 @@ namespace cryptonote m_txpool_auto_relayer.do_call(boost::bind(&core::relay_txpool_transactions, this)); m_check_updates_interval.do_call(boost::bind(&core::check_updates, this)); m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this)); + m_block_rate_interval.do_call(boost::bind(&core::check_block_rate, this)); m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -1429,10 +1565,14 @@ namespace cryptonote return false; if (tools::vercmp(version.c_str(), MONERO_VERSION) <= 0) + { + m_update_available = false; return true; + } std::string url = tools::get_update_url(software, subdir, buildtag, version, true); MCLOG_CYAN(el::Level::Info, "global", "Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash); + m_update_available = true; if (check_updates_level == UPDATES_NOTIFY) return true; @@ -1543,6 +1683,52 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + double factorial(unsigned int n) + { + if (n <= 1) + return 1.0; + double f = n; + while (n-- > 1) + f *= n; + return f; + } + //----------------------------------------------------------------------------------------------- + static double probability(unsigned int blocks, unsigned int expected) + { + // https://www.umass.edu/wsp/resources/poisson/#computing + return pow(expected, blocks) / (factorial(blocks) * exp(expected)); + } + //----------------------------------------------------------------------------------------------- + bool core::check_block_rate() + { + if (m_offline || m_target_blockchain_height > get_current_blockchain_height()) + { + MDEBUG("Not checking block rate, offline or syncing"); + return true; + } + + static constexpr double threshold = 1. / (864000 / DIFFICULTY_TARGET_V2); // one false positive every 10 days + + const time_t now = time(NULL); + const std::vector<time_t> timestamps = m_blockchain_storage.get_last_block_timestamps(60); + + static const unsigned int seconds[] = { 5400, 1800, 600 }; + for (size_t n = 0; n < sizeof(seconds)/sizeof(seconds[0]); ++n) + { + unsigned int b = 0; + for (time_t ts: timestamps) b += ts >= now - static_cast<time_t>(seconds[n]); + const double p = probability(b, seconds[n] / DIFFICULTY_TARGET_V2); + MDEBUG("blocks in the last " << seconds[n] / 60 << " minutes: " << b << " (probability " << p << ")"); + if (p < threshold) + { + MWARNING("There were " << b << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Monero network or under attack. Or it could be just sheer bad luck."); + break; // no need to look further + } + } + + return true; + } + //----------------------------------------------------------------------------------------------- void core::set_target_blockchain_height(uint64_t target_blockchain_height) { m_target_blockchain_height = target_blockchain_height; @@ -1553,7 +1739,7 @@ namespace cryptonote return m_target_blockchain_height; } //----------------------------------------------------------------------------------------------- - uint64_t core::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) + uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return get_blockchain_storage().prevalidate_block_hashes(height, hashes); } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 567966d48..4eca2a57b 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -39,7 +39,6 @@ #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "common/download.h" -#include "common/threadpool.h" #include "common/command_line.h" #include "tx_pool.h" #include "blockchain.h" @@ -61,6 +60,8 @@ namespace cryptonote extern const command_line::arg_descriptor<std::string, false, true, 2> arg_data_dir; extern const command_line::arg_descriptor<bool, false> arg_testnet_on; extern const command_line::arg_descriptor<bool, false> arg_stagenet_on; + extern const command_line::arg_descriptor<bool, false> arg_regtest_on; + extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty; extern const command_line::arg_descriptor<bool> arg_offline; /************************************************************************/ @@ -116,7 +117,7 @@ namespace cryptonote * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * - * @return true if the transaction made it to the transaction pool, otherwise false + * @return true if the transaction was accepted, false otherwise */ bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); @@ -132,9 +133,9 @@ namespace cryptonote * @param relayed whether or not the transactions were relayed to us * @param do_not_relay whether to prevent the transactions from being relayed * - * @return true if the transactions made it to the transaction pool, otherwise false + * @return true if the transactions were accepted, false otherwise */ - bool handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief handles an incoming block @@ -157,7 +158,7 @@ namespace cryptonote * * @note see Blockchain::prepare_handle_incoming_blocks */ - bool prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); /** * @copydoc Blockchain::cleanup_handle_incoming_blocks @@ -243,10 +244,11 @@ namespace cryptonote * @param vm command line parameters * @param config_subdir subdirectory for config storage * @param test_options configuration options for testing + * @param get_checkpoints if set, will be called to get checkpoints data, must return checkpoints data pointer and size or nullptr if there ain't any checkpoints for specific network type * * @return false if one of the init steps fails, otherwise true */ - bool init(const boost::program_options::variables_map& vm, const char *config_subdir = NULL, const test_options *test_options = NULL); + bool init(const boost::program_options::variables_map& vm, const char *config_subdir = NULL, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr); /** * @copydoc Blockchain::reset_and_set_genesis_block @@ -309,25 +311,25 @@ namespace cryptonote void get_blockchain_top(uint64_t& height, crypto::hash& top_id) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&, std::list<transaction>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&, std::vector<transaction>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&, std::list<transaction>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&, std::vector<transaction>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<block>& blocks) const; /** * @copydoc Blockchain::get_blocks(const t_ids_container&, t_blocks_container&, t_missed_container&) const @@ -352,14 +354,14 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::blobdata>& txs, std::list<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const; /** * @copydoc Blockchain::get_transactions * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; /** * @copydoc Blockchain::get_block_by_hash @@ -371,9 +373,9 @@ namespace cryptonote /** * @copydoc Blockchain::get_alternative_blocks * - * @note see Blockchain::get_alternative_blocks(std::list<block>&) const + * @note see Blockchain::get_alternative_blocks(std::vector<block>&) const */ - bool get_alternative_blocks(std::list<block>& blocks) const; + bool get_alternative_blocks(std::vector<block>& blocks) const; /** * @copydoc Blockchain::get_alternative_blocks_count @@ -430,7 +432,7 @@ namespace cryptonote * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transactions(std::list<transaction>& txs, bool include_unrelayed_txes = true) const; + bool get_pool_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; /** * @copydoc tx_memory_pool::get_txpool_backlog @@ -513,11 +515,11 @@ namespace cryptonote bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; /** - * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const + * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const * - * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<cryptonote::blobdata, std::list<transaction> > >&, uint64_t&, uint64_t&, size_t) const + * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<transaction> > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief gets some stats about the daemon @@ -550,13 +552,6 @@ namespace cryptonote difficulty_type get_block_cumulative_difficulty(uint64_t height) const; /** - * @copydoc Blockchain::get_random_outs_for_amounts - * - * @note see Blockchain::get_random_outs_for_amounts - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @copydoc Blockchain::get_outs * * @note see Blockchain::get_outs @@ -564,14 +559,6 @@ namespace cryptonote bool get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const; /** - * - * @copydoc Blockchain::get_random_rct_outs - * - * @note see Blockchain::get_random_rct_outs - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @copydoc Blockchain::get_output_distribution * * @brief get per block distribution of outputs of a given amount @@ -753,6 +740,16 @@ namespace cryptonote network_type get_nettype() const { return m_nettype; }; /** + * @brief check whether an update is known to be available or not + * + * This does not actually trigger a check, but returns the result + * of the last check + * + * @return whether an update is known to be available or not + */ + bool is_update_available() const { return m_update_available; } + + /** * @brief get whether fluffy blocks are enabled * * @return whether fluffy blocks are enabled @@ -764,7 +761,7 @@ namespace cryptonote * * @return number of usable blocks */ - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); /** * @brief get free disk space on the blockchain partition @@ -787,12 +784,12 @@ namespace cryptonote * * @param tx_hash the transaction's hash * @param tx_prefix_hash the transaction prefix' hash - * @param blob_size the size of the transaction + * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -863,9 +860,12 @@ namespace cryptonote * @return true if all the checks pass, otherwise false */ bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; + void set_semantics_failed(const crypto::hash &tx_hash); bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; + bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block); /** * @copydoc miner::on_block_chain_update @@ -946,6 +946,13 @@ namespace cryptonote */ bool check_disk_space(); + /** + * @brief checks block rate, and warns if it's too slow + * + * @return true on success, false otherwise + */ + bool check_block_rate(); + bool m_test_drop_download = true; //!< whether or not to drop incoming blocks (for testing) uint64_t m_test_drop_download_height = 0; //!< height under which to drop incoming blocks, if doing so @@ -970,6 +977,7 @@ namespace cryptonote epee::math_helper::once_a_time_seconds<60*2, false> m_txpool_auto_relayer; //!< interval for checking re-relaying txpool transactions epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions epee::math_helper::once_a_time_seconds<60*10, true> m_check_disk_space_interval; //!< interval for checking for disk space + epee::math_helper::once_a_time_seconds<90, false> m_block_rate_interval; //!< interval for checking block rate std::atomic<bool> m_starter_message_showed; //!< has the "daemon will sync now" message been shown? @@ -977,6 +985,8 @@ namespace cryptonote network_type m_nettype; //!< which network are we on? + std::atomic<bool> m_update_available; + std::string m_checkpoints_path; //!< path to json checkpoints file time_t m_last_dns_checkpoints_update; //!< time when dns checkpoints were last updated time_t m_last_json_checkpoints_update; //!< time when json checkpoints were last updated @@ -991,8 +1001,6 @@ namespace cryptonote std::unordered_set<crypto::hash> bad_semantics_txes[2]; boost::mutex bad_semantics_txes_lock; - tools::threadpool& m_threadpool; - enum { UPDATES_DISABLED, UPDATES_NOTIFY, diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 071ce591e..4fc2736a6 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -38,6 +38,7 @@ using namespace epee; #include "cryptonote_tx_utils.h" #include "cryptonote_config.h" #include "cryptonote_basic/miner.h" +#include "cryptonote_basic/tx_extra.h" #include "crypto/crypto.h" #include "crypto/hash.h" #include "ringct/rctSigs.h" @@ -74,7 +75,7 @@ namespace cryptonote LOG_PRINT_L2("destinations include " << num_stdaddresses << " standard addresses and " << num_subaddresses << " subaddresses"); } //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { tx.vin.clear(); tx.vout.clear(); tx.extra.clear(); @@ -84,12 +85,14 @@ namespace cryptonote if(!extra_nonce.empty()) if(!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) return false; + if (!sort_tx_extra(tx.extra, tx.extra)) + return false; txin_gen in; in.height = height; uint64_t block_reward; - if(!get_block_reward(median_size, current_block_size, already_generated_coins, block_reward, hard_fork_version)) + if(!get_block_reward(median_weight, current_block_weight, already_generated_coins, block_reward, hard_fork_version)) { LOG_PRINT_L0("Block is too big"); return false; @@ -127,7 +130,7 @@ namespace cryptonote out_amounts[1] += out_amounts[0]; for (size_t n = 1; n < out_amounts.size(); ++n) out_amounts[n - 1] = out_amounts[n]; - out_amounts.resize(out_amounts.size() - 1); + out_amounts.pop_back(); } } else @@ -195,7 +198,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -434,6 +437,9 @@ namespace cryptonote add_additional_tx_pub_keys_to_extra(tx.extra, additional_tx_public_keys); } + if (!sort_tx_extra(tx.extra, tx.extra)) + return false; + //check money if(summary_outs_money > summary_inputs_money ) { @@ -491,7 +497,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1; + bool use_simple_rct = sources.size() > 1 || range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -516,6 +522,7 @@ namespace cryptonote uint64_t amount_in = 0, amount_out = 0; rct::ctkeyV inSk; + inSk.reserve(sources.size()); // mixRing indexing is done the other way round for simple rct::ctkeyM mixRing(use_simple_rct ? sources.size() : n_total_outs); rct::keyV destinations; @@ -532,6 +539,7 @@ namespace cryptonote ctkey.dest = rct::sk2rct(in_contexts[i].in_ephemeral.sec); ctkey.mask = sources[i].mask; inSk.push_back(ctkey); + memwipe(&ctkey, sizeof(rct::ctkey)); // inPk: (public key, commitment) // will be done when filling in mixRing if (msout) @@ -587,9 +595,10 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, bulletproof, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, range_proof_type, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, bulletproof, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, hwdev); // same index assumption + memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -601,7 +610,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -619,7 +628,7 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, bulletproof, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, range_proof_type, msout); hwdev.close_tx(); return r; } @@ -631,7 +640,7 @@ namespace cryptonote crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index a5d149fca..f2cf7b6ca 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -37,7 +37,7 @@ namespace cryptonote { //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); struct tx_source_entry { @@ -90,8 +90,8 @@ namespace cryptonote //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL); bool generate_genesis_block( block& bl diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index bf1fe476e..70f7e8328 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -80,9 +80,13 @@ namespace cryptonote return amount * ACCEPT_THRESHOLD; } - uint64_t get_transaction_size_limit(uint8_t version) + uint64_t get_transaction_weight_limit(uint8_t version) { - return get_min_block_size(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + // from v8, limit a tx to 50% of the minimum block weight + if (version >= 8) + return get_min_block_weight(version) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return get_min_block_weight(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } // This class is meant to create a batch when none currently exists. @@ -102,12 +106,12 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //--------------------------------------------------------------------------------- - tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_size(DEFAULT_TXPOOL_MAX_SIZE), m_txpool_size(0) + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_cookie(0) { } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -173,17 +177,17 @@ namespace cryptonote fee = tx.rct_signatures.txnFee; } - if (!kept_by_block && !m_blockchain.check_fee(blob_size, fee)) + if (!kept_by_block && !m_blockchain.check_fee(tx_weight, fee)) { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; return false; } - size_t tx_size_limit = get_transaction_size_limit(version); - if (!kept_by_block && blob_size > tx_size_limit) + size_t tx_weight_limit = get_transaction_weight_limit(version); + if ((!kept_by_block || version >= HF_VERSION_PER_BYTE_FEE) && tx_weight > tx_weight_limit) { - LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit); + LOG_PRINT_L1("transaction is too heavy: " << tx_weight << " bytes, maximum weight: " << tx_weight_limit); tvc.m_verifivation_failed = true; tvc.m_too_big = true; return false; @@ -220,14 +224,14 @@ namespace cryptonote crypto::hash max_used_block_id = null_hash; uint64_t max_used_block_height = 0; cryptonote::txpool_tx_meta_t meta; - bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, tvc, kept_by_block); + bool ch_inp_res = check_tx_inputs([&tx]()->cryptonote::transaction&{ return tx; }, id, max_used_block_height, max_used_block_id, tvc, kept_by_block); if(!ch_inp_res) { // if the transaction was valid before (kept_by_block), then it // may become valid again, so ignore the failed inputs check. if(kept_by_block) { - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.fee = fee; meta.max_used_block_id = null_hash; meta.max_used_block_height = 0; @@ -239,15 +243,16 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = have_tx_keyimges_as_spent(tx); + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try { CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); m_blockchain.add_txpool_tx(tx, meta); - if (!insert_key_images(tx, kept_by_block)) + if (!insert_key_images(tx, id, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -266,7 +271,7 @@ namespace cryptonote }else { //update transactions container - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.kept_by_block = kept_by_block; meta.fee = fee; meta.max_used_block_id = max_used_block_id; @@ -278,17 +283,19 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = false; + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try { CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - m_blockchain.remove_txpool_tx(get_transaction_hash(tx)); + const crypto::hash txid = get_transaction_hash(tx); + m_blockchain.remove_txpool_tx(txid); m_blockchain.add_txpool_tx(tx, meta); - if (!insert_key_images(tx, kept_by_block)) + if (!insert_key_images(tx, txid, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -302,11 +309,13 @@ namespace cryptonote } tvc.m_verifivation_failed = false; - m_txpool_size += blob_size; + m_txpool_weight += tx_weight; + + ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size)); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)tx_weight)); - prune(m_txpool_max_size); + prune(m_txpool_max_weight); return true; } @@ -317,34 +326,35 @@ namespace cryptonote size_t blob_size = 0; if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0) return false; - return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, get_transaction_weight(tx, blob_size), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- - size_t tx_memory_pool::get_txpool_size() const + size_t tx_memory_pool::get_txpool_weight() const { CRITICAL_REGION_LOCAL(m_transactions_lock); - return m_txpool_size; + return m_txpool_weight; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_txpool_max_size(size_t bytes) + void tx_memory_pool::set_txpool_max_weight(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); - m_txpool_max_size = bytes; + m_txpool_max_weight = bytes; } //--------------------------------------------------------------------------------- void tx_memory_pool::prune(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); if (bytes == 0) - bytes = m_txpool_max_size; + bytes = m_txpool_max_weight; CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); + bool changed = false; // this will never remove the first one, but we don't care auto it = --m_txs_by_fee_and_receive_time.end(); while (it != m_txs_by_fee_and_receive_time.begin()) { - if (m_txpool_size <= bytes) + if (m_txpool_weight <= bytes) break; try { @@ -362,19 +372,20 @@ namespace cryptonote continue; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid); - cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) + cryptonote::transaction_prefix tx; + if (!parse_and_validate_tx_prefix_from_blob(txblob, tx)) { MERROR("Failed to parse tx from txpool"); return; } // remove first, in case this throws, so key images aren't removed - MINFO("Pruning tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruning tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); - remove_transaction_keyimages(tx); - MINFO("Pruned tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + m_txpool_weight -= meta.weight; + remove_transaction_keyimages(tx, txid); + MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_txs_by_fee_and_receive_time.erase(it--); + changed = true; } catch (const std::exception &e) { @@ -382,15 +393,16 @@ namespace cryptonote return; } } - if (m_txpool_size > bytes) - MINFO("Pool size after pruning is larger than limit: " << m_txpool_size << "/" << bytes); + if (changed) + ++m_cookie; + if (m_txpool_weight > bytes) + MINFO("Pool weight after pruning is larger than limit: " << m_txpool_weight << "/" << bytes); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::insert_key_images(const transaction &tx, bool kept_by_block) + bool tx_memory_pool::insert_key_images(const transaction_prefix &tx, const crypto::hash &id, bool kept_by_block) { for(const auto& in: tx.vin) { - const crypto::hash id = get_transaction_hash(tx); CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false); std::unordered_set<crypto::hash>& kei_image_set = m_spent_key_images[txin.k_image]; CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block @@ -399,25 +411,24 @@ namespace cryptonote auto ins_res = kei_image_set.insert(id); CHECK_AND_ASSERT_MES(ins_res.second, false, "internal error: try to insert duplicate iterator in key_image set"); } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- //FIXME: Can return early before removal of all of the key images. // At the least, need to make sure that a false return here // is treated properly. Should probably not return early, however. - bool tx_memory_pool::remove_transaction_keyimages(const transaction& tx) + bool tx_memory_pool::remove_transaction_keyimages(const transaction_prefix& tx, const crypto::hash &actual_hash) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); // ND: Speedup - // 1. Move transaction hash calcuation outside of loop. ._. - crypto::hash actual_hash = get_transaction_hash(tx); for(const txin_v& vi: tx.vin) { CHECKED_GET_SPECIFIC_VARIANT(vi, const txin_to_key, txin, false); auto it = m_spent_key_images.find(txin.k_image); CHECK_AND_ASSERT_MES(it != m_spent_key_images.end(), false, "failed to find transaction input in key images. img=" << txin.k_image << ENDL - << "transaction id = " << get_transaction_hash(tx)); + << "transaction id = " << actual_hash); std::unordered_set<crypto::hash>& key_image_set = it->second; CHECK_AND_ASSERT_MES(key_image_set.size(), false, "empty key_image set, img=" << txin.k_image << ENDL << "transaction id = " << actual_hash); @@ -433,10 +444,11 @@ namespace cryptonote } } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) + bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -460,7 +472,7 @@ namespace cryptonote MERROR("Failed to parse tx from txpool"); return false; } - blob_size = meta.blob_size; + tx_weight = meta.weight; fee = meta.fee; relayed = meta.relayed; do_not_relay = meta.do_not_relay; @@ -468,8 +480,8 @@ namespace cryptonote // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); - m_txpool_size -= blob_size; - remove_transaction_keyimages(tx); + m_txpool_weight -= tx_weight; + remove_transaction_keyimages(tx, id); } catch (const std::exception &e) { @@ -478,6 +490,7 @@ namespace cryptonote } m_txs_by_fee_and_receive_time.erase(sorted_it); + ++m_cookie; return true; } //--------------------------------------------------------------------------------- @@ -500,7 +513,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - std::unordered_set<crypto::hash> remove; + std::list<std::pair<crypto::hash, uint64_t>> remove; m_blockchain.for_all_txpool_txes([this, &remove](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { uint64_t tx_age = time(nullptr) - meta.receive_time; @@ -518,7 +531,7 @@ namespace cryptonote m_txs_by_fee_and_receive_time.erase(sorted_it); } m_timed_out_transactions.insert(txid); - remove.insert(txid); + remove.push_back(std::make_pair(txid, meta.weight)); } return true; }, false); @@ -526,13 +539,14 @@ namespace cryptonote if (!remove.empty()) { LockedTXN lock(m_blockchain); - for (const crypto::hash &txid: remove) + for (const std::pair<crypto::hash, uint64_t> &entry: remove) { + const crypto::hash &txid = entry.first; try { cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid); - cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) + cryptonote::transaction_prefix tx; + if (!parse_and_validate_tx_prefix_from_blob(bd, tx)) { MERROR("Failed to parse tx from txpool"); // continue @@ -541,8 +555,8 @@ namespace cryptonote { // remove first, so we only remove key images if the tx removal succeeds m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= bd.size(); - remove_transaction_keyimages(tx); + m_txpool_weight -= entry.second; + remove_transaction_keyimages(tx, txid); } } catch (const std::exception &e) @@ -551,16 +565,18 @@ namespace cryptonote // ignore error } } + ++m_cookie; } return true; } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const + bool tx_memory_pool::get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + txs.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) @@ -588,7 +604,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs) + void tx_memory_pool::set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -621,10 +637,11 @@ namespace cryptonote return m_blockchain.get_txpool_tx_count(include_unrelayed_txes); } //--------------------------------------------------------------------------------- - void tx_memory_pool::get_transactions(std::list<transaction>& txs, bool include_unrelayed_txes) const + void tx_memory_pool::get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ transaction tx; if (!parse_and_validate_tx_from_blob(*bd, tx)) @@ -642,6 +659,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ txs.push_back(txid); return true; @@ -653,8 +671,9 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); + backlog.push_back({meta.weight, meta.fee, meta.receive_time - now}); return true; }, false, include_unrelayed_txes); } @@ -666,15 +685,15 @@ namespace cryptonote const uint64_t now = time(NULL); std::map<uint64_t, txpool_histo> agebytes; stats.txs_total = m_blockchain.get_txpool_tx_count(include_unrelayed_txes); - std::vector<uint32_t> sizes; - sizes.reserve(stats.txs_total); - m_blockchain.for_all_txpool_txes([&stats, &sizes, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - sizes.push_back(meta.blob_size); - stats.bytes_total += meta.blob_size; - if (!stats.bytes_min || meta.blob_size < stats.bytes_min) - stats.bytes_min = meta.blob_size; - if (meta.blob_size > stats.bytes_max) - stats.bytes_max = meta.blob_size; + std::vector<uint32_t> weights; + weights.reserve(stats.txs_total); + m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + weights.push_back(meta.weight); + stats.bytes_total += meta.weight; + if (!stats.bytes_min || meta.weight < stats.bytes_min) + stats.bytes_min = meta.weight; + if (meta.weight > stats.bytes_max) + stats.bytes_max = meta.weight; if (!meta.relayed) stats.num_not_relayed++; stats.fee_total += meta.fee; @@ -686,12 +705,12 @@ namespace cryptonote stats.num_failing++; uint64_t age = now - meta.receive_time + (now == meta.receive_time); agebytes[age].txs++; - agebytes[age].bytes += meta.blob_size; + agebytes[age].bytes += meta.weight; if (meta.double_spend_seen) ++stats.num_double_spends; return true; }, false, include_unrelayed_txes); - stats.bytes_med = epee::misc_utils::median(sizes); + stats.bytes_med = epee::misc_utils::median(weights); if (stats.txs_total > 1) { /* looking for 98th percentile */ @@ -741,6 +760,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ tx_info txi; txi.id_hash = epee::string_tools::pod_to_hex(txid); @@ -753,7 +774,8 @@ namespace cryptonote return true; } txi.tx_json = obj_to_json_str(tx); - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -811,6 +833,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; @@ -822,7 +846,8 @@ namespace cryptonote return true; } txi.tx = tx; - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -883,11 +908,15 @@ namespace cryptonote //--------------------------------------------------------------------------------- bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) { + CRITICAL_REGION_LOCAL(m_transactions_lock); + m_input_cache.clear(); return true; } //--------------------------------------------------------------------------------- bool tx_memory_pool::on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id) { + CRITICAL_REGION_LOCAL(m_transactions_lock); + m_input_cache.clear(); return true; } //--------------------------------------------------------------------------------- @@ -927,8 +956,45 @@ namespace cryptonote m_transactions_lock.unlock(); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const + bool tx_memory_pool::check_tx_inputs(const std::function<cryptonote::transaction&(void)> &get_tx, const crypto::hash &txid, uint64_t &max_used_block_height, crypto::hash &max_used_block_id, tx_verification_context &tvc, bool kept_by_block) const { + if (!kept_by_block) + { + const std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>>::const_iterator i = m_input_cache.find(txid); + if (i != m_input_cache.end()) + { + max_used_block_height = std::get<2>(i->second); + max_used_block_id = std::get<3>(i->second); + tvc = std::get<1>(i->second); + return std::get<0>(i->second); + } + } + bool ret = m_blockchain.check_tx_inputs(get_tx(), max_used_block_height, max_used_block_id, tvc, kept_by_block); + if (!kept_by_block) + m_input_cache.insert(std::make_pair(txid, std::make_tuple(ret, tvc, max_used_block_height, max_used_block_id))); + return ret; + } + //--------------------------------------------------------------------------------- + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const + { + struct transction_parser + { + transction_parser(const cryptonote::blobdata &txblob, transaction &tx): txblob(txblob), tx(tx), parsed(false) {} + cryptonote::transaction &operator()() + { + if (!parsed) + { + if (!parse_and_validate_tx_from_blob(txblob, tx)) + throw std::runtime_error("failed to parse transaction blob"); + parsed = true; + } + return tx; + } + const cryptonote::blobdata &txblob; + transaction &tx; + bool parsed; + } lazy_tx(txblob, tx); + //not the best implementation at this time, sorry :( //check is ring_signature already checked ? if(txd.max_used_block_id == null_hash) @@ -938,7 +1004,7 @@ namespace cryptonote return false;//we already sure that this tx is broken for this height tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -955,7 +1021,7 @@ namespace cryptonote return false; //check ring signature again, it is possible (with very small chance) that this transaction become again valid tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -964,7 +1030,7 @@ namespace cryptonote } } //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure - if(m_blockchain.have_tx_keyimges_as_spent(tx)) + if(m_blockchain.have_tx_keyimges_as_spent(lazy_tx())) { txd.double_spend_seen = true; return false; @@ -974,7 +1040,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::have_key_images(const std::unordered_set<crypto::key_image>& k_images, const transaction& tx) + bool tx_memory_pool::have_key_images(const std::unordered_set<crypto::key_image>& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -985,7 +1051,7 @@ namespace cryptonote return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::append_key_images(std::unordered_set<crypto::key_image>& k_images, const transaction& tx) + bool tx_memory_pool::append_key_images(std::unordered_set<crypto::key_image>& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -1000,6 +1066,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + bool changed = false; LockedTXN lock(m_blockchain); for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -1020,6 +1087,7 @@ namespace cryptonote { MDEBUG("Marking " << txid << " as double spending " << itk.k_image); meta.double_spend_seen = true; + changed = true; try { m_blockchain.update_txpool_tx(txid, meta); @@ -1033,6 +1101,8 @@ namespace cryptonote } } } + if (changed) + ++m_cookie; } //--------------------------------------------------------------------------------- std::string tx_memory_pool::print_pool(bool short_format) const @@ -1051,7 +1121,8 @@ namespace cryptonote } ss << obj_to_json_str(tx) << std::endl; } - ss << "blob_size: " << meta.blob_size << std::endl + ss << "blob_size: " << (short_format ? "-" : std::to_string(txblob->size())) << std::endl + << "weight: " << meta.weight << std::endl << "fee: " << print_money(meta.fee) << std::endl << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl << "double_spend_seen: " << (meta.double_spend_seen ? 'T' : 'F') << std::endl @@ -1066,30 +1137,30 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version) + bool tx_memory_pool::fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); uint64_t best_coinbase = 0, coinbase = 0; - total_size = 0; + total_weight = 0; fee = 0; //baseline empty block - get_block_reward(median_size, total_size, already_generated_coins, best_coinbase, version); + get_block_reward(median_weight, total_weight, already_generated_coins, best_coinbase, version); - size_t max_total_size_pre_v5 = (130 * median_size) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size_v5 = 2 * median_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size = version >= 5 ? max_total_size_v5 : max_total_size_pre_v5; + size_t max_total_weight_pre_v5 = (130 * median_weight) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight_v5 = 2 * median_weight - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight = version >= 5 ? max_total_weight_v5 : max_total_weight_pre_v5; std::unordered_set<crypto::key_image> k_images; - LOG_PRINT_L2("Filling block template, median size " << median_size << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); + LOG_PRINT_L2("Filling block template, median weight " << median_weight << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); LockedTXN lock(m_blockchain); auto sorted_it = m_txs_by_fee_and_receive_time.begin(); - while (sorted_it != m_txs_by_fee_and_receive_time.end()) + for (; sorted_it != m_txs_by_fee_and_receive_time.end(); ++sorted_it) { txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta)) @@ -1097,13 +1168,12 @@ namespace cryptonote MERROR(" failed to find tx meta"); continue; } - LOG_PRINT_L2("Considering " << sorted_it->second << ", size " << meta.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase)); - // Can not exceed maximum block size - if (max_total_size < total_size + meta.blob_size) + // Can not exceed maximum block weight + if (max_total_weight < total_weight + meta.weight) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } @@ -1113,45 +1183,46 @@ namespace cryptonote // If we're getting lower coinbase tx, // stop including more tx uint64_t block_reward; - if(!get_block_reward(median_size, total_size + meta.blob_size, already_generated_coins, block_reward, version)) + if(!get_block_reward(median_weight, total_weight + meta.weight, already_generated_coins, block_reward, version)) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } coinbase = block_reward + fee + meta.fee; if (coinbase < template_accept_threshold(best_coinbase)) { LOG_PRINT_L2(" would decrease coinbase to " << print_money(coinbase)); - sorted_it++; continue; } } else { - // If we've exceeded the penalty free size, + // If we've exceeded the penalty free weight, // stop including more tx - if (total_size > median_size) + if (total_weight > median_weight) { - LOG_PRINT_L2(" would exceed median block size"); + LOG_PRINT_L2(" would exceed median block weight"); break; } } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second); cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) - { - MERROR("Failed to parse tx from txpool"); - sorted_it++; - continue; - } // Skip transactions that are not ready to be // included into the blockchain or that are // missing key images const cryptonote::txpool_tx_meta_t original_meta = meta; - bool ready = is_transaction_ready_to_go(meta, tx); + bool ready = false; + try + { + ready = is_transaction_ready_to_go(meta, sorted_it->second, txblob, tx); + } + catch (const std::exception &e) + { + MERROR("Failed to check transaction readiness: " << e.what()); + // continue, not fatal + } if (memcmp(&original_meta, &meta, sizeof(meta))) { try @@ -1167,28 +1238,25 @@ namespace cryptonote if (!ready) { LOG_PRINT_L2(" not ready to go"); - sorted_it++; continue; } if (have_key_images(k_images, tx)) { LOG_PRINT_L2(" key images already seen"); - sorted_it++; continue; } bl.tx_hashes.push_back(sorted_it->second); - total_size += meta.blob_size; + total_weight += meta.weight; fee += meta.fee; best_coinbase = coinbase; append_key_images(k_images, tx); - sorted_it++; - LOG_PRINT_L2(" added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2(" added, new block weight " << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase)); } expected_reward = best_coinbase; - LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, size " - << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase) + LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, weight " + << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase) << " (including " << print_money(fee) << " in fees)"); return true; } @@ -1197,14 +1265,14 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - size_t tx_size_limit = get_transaction_size_limit(version); + size_t tx_weight_limit = get_transaction_weight_limit(version); std::unordered_set<crypto::hash> remove; - m_txpool_size = 0; - m_blockchain.for_all_txpool_txes([this, &remove, tx_size_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { - m_txpool_size += meta.blob_size; - if (meta.blob_size > tx_size_limit) { - LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.blob_size << " bytes), removing it from pool"); + m_txpool_weight = 0; + m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_txpool_weight += meta.weight; + if (meta.weight > tx_weight_limit) { + LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.weight << " bytes), removing it from pool"); remove.insert(txid); } else if (m_blockchain.have_tx(txid)) { @@ -1231,8 +1299,8 @@ namespace cryptonote } // remove tx from db first m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); - remove_transaction_keyimages(tx); + m_txpool_weight -= get_transaction_weight(tx, txblob.size()); + remove_transaction_keyimages(tx, txid); auto sorted_it = find_tx_in_sorted_container(txid); if (sorted_it == m_txs_by_fee_and_receive_time.end()) { @@ -1251,18 +1319,20 @@ namespace cryptonote } } } + if (n_removed > 0) + ++m_cookie; return n_removed; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::init(size_t max_txpool_size) + bool tx_memory_pool::init(size_t max_txpool_weight) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - m_txpool_max_size = max_txpool_size ? max_txpool_size : DEFAULT_TXPOOL_MAX_SIZE; + m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); m_spent_key_images.clear(); - m_txpool_size = 0; + m_txpool_weight = 0; std::vector<crypto::hash> remove; // first add the not kept by block, then the kept by block, @@ -1273,19 +1343,20 @@ namespace cryptonote bool r = m_blockchain.for_all_txpool_txes([this, &remove, kept](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd) { if (!!kept != !!meta.kept_by_block) return true; - cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(*bd, tx)) + cryptonote::transaction_prefix tx; + if (!parse_and_validate_tx_prefix_from_blob(*bd, tx)) { MWARNING("Failed to parse tx from txpool, removing"); remove.push_back(txid); + return true; } - if (!insert_key_images(tx, meta.kept_by_block)) + if (!insert_key_images(tx, txid, meta.kept_by_block)) { MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.blob_size, meta.receive_time), txid); - m_txpool_size += meta.blob_size; + m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid); + m_txpool_weight += meta.weight; return true; }, true); if (!r) @@ -1307,6 +1378,10 @@ namespace cryptonote } } } + + m_cookie = 0; + + // Ignore deserialization error return true; } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 19cd83ed9..7a0cc23bf 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -84,7 +84,7 @@ namespace cryptonote * * This handling includes: * storing the transactions - * organizing the transactions by fee per size + * organizing the transactions by fee per weight unit * taking/giving transactions to and from various other components * saving the transactions to disk on shutdown * helping create a new block template by choosing transactions for it @@ -105,9 +105,9 @@ namespace cryptonote * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) * * @param id the transaction's hash - * @param blob_size the transaction's size + * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -133,7 +133,7 @@ namespace cryptonote * * @param id the hash of the transaction * @param tx return-by-reference the transaction taken - * @param blob_size return-by-reference the transaction's size + * @param tx_weight return-by-reference the transaction's weight * @param fee the transaction fee * @param relayed return-by-reference was transaction relayed to us by the network? * @param do_not_relay return-by-reference is transaction not to be relayed to the network? @@ -141,7 +141,7 @@ namespace cryptonote * * @return true unless the transaction cannot be found in the pool */ - bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); + bool take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); /** * @brief checks if the pool has a transaction with the given hash @@ -198,11 +198,11 @@ namespace cryptonote /** * @brief loads pool state (if any) from disk, and initializes pool * - * @param max_txpool_size the max size in bytes + * @param max_txpool_weight the max weight in bytes * * @return true */ - bool init(size_t max_txpool_size = 0); + bool init(size_t max_txpool_weight = 0); /** * @brief attempts to save the transaction pool state to disk @@ -219,16 +219,16 @@ namespace cryptonote * @brief Chooses transactions for a block to include * * @param bl return-by-reference the block to fill in with transactions - * @param median_size the current median block size + * @param median_weight the current median block weight * @param already_generated_coins the current total number of coins "minted" - * @param total_size return-by-reference the total size of the new block + * @param total_weight return-by-reference the total weight of the new block * @param fee return-by-reference the total of fees from the included transactions * @param expected_reward return-by-reference the total reward awarded to the miner finding this block, including transaction fees * @param version hard fork version to use for consensus rules * * @return true */ - bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version); + bool fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version); /** * @brief get a list of all transactions in the pool @@ -237,7 +237,7 @@ namespace cryptonote * @param include_unrelayed_txes include unrelayed txes in the result * */ - void get_transactions(std::list<transaction>& txs, bool include_unrelayed_txes = true) const; + void get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; /** * @brief get a list of all transaction hashes in the pool @@ -249,7 +249,7 @@ namespace cryptonote void get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes = true) const; /** - * @brief get (size, fee, receive time) for all transaction in the pool + * @brief get (weight, fee, receive time) for all transaction in the pool * * @param txs return-by-reference that data * @param include_unrelayed_txes include unrelayed txes in the result @@ -324,14 +324,14 @@ namespace cryptonote * * @return true */ - bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const; + bool get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const; /** * @brief tell the pool that certain transactions were just relayed * * @param txs the list of transactions (and their hashes) */ - void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs); + void set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs); /** * @brief get the total number of transactions in the pool @@ -362,22 +362,29 @@ namespace cryptonote */ size_t validate(uint8_t version); + /** + * @brief return the cookie + * + * @return the cookie + */ + uint64_t cookie() const { return m_cookie; } + /** - * @brief get the cumulative txpool size in bytes + * @brief get the cumulative txpool weight in bytes * - * @return the cumulative txpool size in bytes + * @return the cumulative txpool weight in bytes */ - size_t get_txpool_size() const; + size_t get_txpool_weight() const; /** - * @brief set the max cumulative txpool size in bytes + * @brief set the max cumulative txpool weight in bytes * - * @param bytes the max cumulative txpool size in bytes + * @param bytes the max cumulative txpool weight in bytes */ - void set_txpool_max_size(size_t bytes); + void set_txpool_max_weight(size_t bytes); #define CURRENT_MEMPOOL_ARCHIVE_VER 11 -#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 12 +#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 13 /** * @brief information about a single transaction @@ -386,6 +393,7 @@ namespace cryptonote { transaction tx; //!< the transaction size_t blob_size; //!< the transaction's size + size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount crypto::hash max_used_block_id; //!< the hash of the highest block referenced by an input uint64_t max_used_block_height; //!< the height of the highest block referenced by an input @@ -426,7 +434,7 @@ namespace cryptonote * * @return true on success, false on error */ - bool insert_key_images(const transaction &tx, bool kept_by_block); + bool insert_key_images(const transaction_prefix &tx, const crypto::hash &txid, bool kept_by_block); /** * @brief remove old transactions from the pool @@ -470,10 +478,11 @@ namespace cryptonote * a transaction from the pool. * * @param tx the transaction + * @param txid the transaction's hash * * @return false if any key images to be removed cannot be found, otherwise true */ - bool remove_transaction_keyimages(const transaction& tx); + bool remove_transaction_keyimages(const transaction_prefix& tx, const crypto::hash &txid); /** * @brief check if any of a transaction's spent key images are present in a given set @@ -483,7 +492,7 @@ namespace cryptonote * * @return true if any key images present in the set, otherwise false */ - static bool have_key_images(const std::unordered_set<crypto::key_image>& kic, const transaction& tx); + static bool have_key_images(const std::unordered_set<crypto::key_image>& kic, const transaction_prefix& tx); /** * @brief append the key images from a transaction to the given set @@ -493,16 +502,19 @@ namespace cryptonote * * @return false if any append fails, otherwise true */ - static bool append_key_images(std::unordered_set<crypto::key_image>& kic, const transaction& tx); + static bool append_key_images(std::unordered_set<crypto::key_image>& kic, const transaction_prefix& tx); /** * @brief check if a transaction is a valid candidate for inclusion in a block * * @param txd the transaction to check (and info about it) + * @param txid the txid of the transaction to check + * @param txblob the transaction blob to check + * @param tx the parsed transaction prefix, if successful * * @return true if the transaction is good to go, otherwise false */ - bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const; + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction&tx) const; /** * @brief mark all transactions double spending the one passed @@ -512,7 +524,7 @@ namespace cryptonote /** * @brief prune lowest fee/byte txes till we're not above bytes * - * if bytes is 0, use m_txpool_max_size + * if bytes is 0, use m_txpool_max_weight */ void prune(size_t bytes = 0); @@ -546,6 +558,8 @@ private: //!< container for transactions organized by fee per size and receive time sorted_tx_container m_txs_by_fee_and_receive_time; + std::atomic<uint64_t> m_cookie; //!< incremented at each change + /** * @brief get an iterator to a transaction in the sorted container * @@ -555,6 +569,9 @@ private: */ sorted_tx_container::iterator find_tx_in_sorted_container(const crypto::hash& id) const; + //! cache/call Blockchain::check_tx_inputs results + bool check_tx_inputs(const std::function<cryptonote::transaction&(void)> &get_tx, const crypto::hash &txid, uint64_t &max_used_block_height, crypto::hash &max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false) const; + //! transactions which are unlikely to be included in blocks /*! These transactions are kept in RAM in case they *are* included * in a block eventually, but this container is not saved to disk. @@ -563,8 +580,10 @@ private: Blockchain& m_blockchain; //!< reference to the Blockchain object - size_t m_txpool_max_size; - size_t m_txpool_size; + size_t m_txpool_max_weight; + size_t m_txpool_weight; + + mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache; }; } @@ -591,6 +610,9 @@ namespace boost if (version < 12) return; ar & td.do_not_relay; + if (version < 13) + return; + ar & td.weight; } } } diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 9ae33d540..05f4189fb 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -50,14 +50,18 @@ namespace std { namespace cryptonote { -void block_queue::add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) +void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) { boost::unique_lock<boost::recursive_mutex> lock(mutex); - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) + { + for (const crypto::hash &h: hashes) + requested_hashes.insert(h); set_span_hashes(height, connection_id, hashes); + } } void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time) @@ -76,11 +80,19 @@ void block_queue::flush_spans(const boost::uuids::uuid &connection_id, bool all) block_map::iterator j = i++; if (j->connection_id == connection_id && (all || j->blocks.size() == 0)) { - blocks.erase(j); + erase_block(j); } } } +void block_queue::erase_block(block_map::iterator j) +{ + CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); + for (const crypto::hash &h: j->hashes) + requested_hashes.erase(h); + blocks.erase(j); +} + void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -92,12 +104,12 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con block_map::iterator j = i++; if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) { - blocks.erase(j); + erase_block(j); } } } -bool block_queue::remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes) +bool block_queue::remove_span(uint64_t start_block_height, std::vector<crypto::hash> *hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -106,7 +118,7 @@ bool block_queue::remove_span(uint64_t start_block_height, std::list<crypto::has { if (hashes) *hashes = std::move(i->hashes); - blocks.erase(i); + erase_block(i); return true; } } @@ -121,7 +133,7 @@ void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t block_map::iterator j = i++; if (j->connection_id == connection_id && j->start_block_height <= start_block_height) { - blocks.erase(j); + erase_block(j); } } } @@ -160,19 +172,18 @@ std::string block_queue::get_overview() const return s; } +inline bool block_queue::requested_internal(const crypto::hash &hash) const +{ + return requested_hashes.find(hash) != requested_hashes.end(); +} + bool block_queue::requested(const crypto::hash &hash) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); - for (const auto &span: blocks) - { - for (const auto &h: span.hashes) - if (h == hash) - return true; - } - return false; + return requested_internal(hash); } -std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list<crypto::hash> &block_hashes, boost::posix_time::ptime time) +std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -183,14 +194,14 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei } uint64_t span_start_height = last_block_height - block_hashes.size() + 1; - std::list<crypto::hash>::const_iterator i = block_hashes.begin(); - while (i != block_hashes.end() && requested(*i)) + std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); + while (i != block_hashes.end() && requested_internal(*i)) { ++i; ++span_start_height; } uint64_t span_length = 0; - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; while (i != block_hashes.end() && span_length < max_blocks) { hashes.push_back(*i); @@ -230,7 +241,7 @@ std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const return std::make_pair(current_height + 1, first_span_height - current_height - 1); } -std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const +std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) @@ -248,7 +259,7 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::list< return std::make_pair(i->start_block_height, i->nblocks); } -void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes) +void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -256,15 +267,17 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui if (i->start_block_height == start_height && i->connection_id == connection_id) { span s = *i; - blocks.erase(i); + erase_block(i); s.hashes = std::move(hashes); + for (const crypto::hash &h: s.hashes) + requested_hashes.insert(h); blocks.insert(s); return; } } } -bool block_queue::get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled) const +bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 69ddaa435..9cce95075 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -31,8 +31,9 @@ #pragma once #include <string> -#include <list> +#include <vector> #include <set> +#include <unordered_set> #include <boost/thread/recursive_mutex.hpp> #include <boost/uuid/uuid.hpp> @@ -49,15 +50,15 @@ namespace cryptonote struct span { uint64_t start_block_height; - std::list<crypto::hash> hashes; - std::list<cryptonote::block_complete_entry> blocks; + std::vector<crypto::hash> hashes; + std::vector<cryptonote::block_complete_entry> blocks; boost::uuids::uuid connection_id; uint64_t nblocks; float rate; size_t size; boost::posix_time::ptime time; - span(uint64_t start_block_height, std::list<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): + span(uint64_t start_block_height, std::vector<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): start_block_height(start_block_height), blocks(std::move(blocks)), connection_id(connection_id), nblocks(this->blocks.size()), rate(rate), size(size), time() {} span(uint64_t start_block_height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time): start_block_height(start_block_height), connection_id(connection_id), nblocks(nblocks), rate(0.0f), size(0), time(time) {} @@ -67,21 +68,21 @@ namespace cryptonote typedef std::set<span> block_map; public: - void add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); + void add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time); void flush_spans(const boost::uuids::uuid &connection_id, bool all = false); void flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections); - bool remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes = NULL); + bool remove_span(uint64_t start_block_height, std::vector<crypto::hash> *hashes = NULL); void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; std::string get_overview() const; - std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); bool is_blockchain_placeholder(const span &span) const; std::pair<uint64_t, uint64_t> get_start_gap_span() const; - std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; - void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes); - bool get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; + std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes); + bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; @@ -93,7 +94,12 @@ namespace cryptonote bool requested(const crypto::hash &hash) const; private: + void erase_block(block_map::iterator j); + inline bool requested_internal(const crypto::hash &hash) const; + + private: block_map blocks; mutable boost::recursive_mutex mutex; + std::unordered_set<crypto::hash> requested_hashes; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index cf0043287..db159f0f4 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -109,7 +109,7 @@ namespace cryptonote struct block_complete_entry { blobdata block; - std::list<blobdata> txs; + std::vector<blobdata> txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block) KV_SERIALIZE(txs) @@ -145,7 +145,7 @@ namespace cryptonote struct request { - std::list<blobdata> txs; + std::vector<blobdata> txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) @@ -161,8 +161,8 @@ namespace cryptonote struct request { - std::list<crypto::hash> txs; - std::list<crypto::hash> blocks; + std::vector<crypto::hash> txs; + std::vector<crypto::hash> blocks; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(txs) @@ -177,9 +177,9 @@ namespace cryptonote struct request { - std::list<blobdata> txs; - std::list<block_complete_entry> blocks; - std::list<crypto::hash> missed_ids; + std::vector<blobdata> txs; + std::vector<block_complete_entry> blocks; + std::vector<crypto::hash> missed_ids; uint64_t current_blockchain_height; BEGIN_KV_SERIALIZE_MAP() @@ -230,7 +230,7 @@ namespace cryptonote uint64_t start_height; uint64_t total_height; uint64_t cumulative_difficulty; - std::list<crypto::hash> m_block_ids; + std::vector<crypto::hash> m_block_ids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2e1df8078..c2c660e8c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -351,7 +351,7 @@ namespace cryptonote return 1; } m_core.pause_mine(); - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; blocks.push_back(arg.b); m_core.prepare_handle_incoming_blocks(blocks); for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) @@ -438,7 +438,7 @@ namespace cryptonote } } - std::list<blobdata> have_tx; + std::vector<blobdata> have_tx; // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -578,8 +578,8 @@ namespace cryptonote else { std::vector<crypto::hash> tx_ids; - std::list<transaction> txes; - std::list<crypto::hash> missing; + std::vector<transaction> txes; + std::vector<crypto::hash> missing; tx_ids.push_back(tx_hash); if (m_core.get_transactions(tx_ids, txes, missing) && missing.empty()) { @@ -626,7 +626,7 @@ namespace cryptonote b.block = arg.b.block; b.txs = have_tx; - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; blocks.push_back(b); m_core.prepare_handle_incoming_blocks(blocks); @@ -687,8 +687,8 @@ namespace cryptonote { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_FLUFFY_MISSING_TX (" << arg.missing_tx_indices.size() << " txes), block hash " << arg.block_hash); - std::list<std::pair<cryptonote::blobdata, block>> local_blocks; - std::list<cryptonote::blobdata> local_txs; + std::vector<std::pair<cryptonote::blobdata, block>> local_blocks; + std::vector<cryptonote::blobdata> local_txs; block b; if (!m_core.get_block_by_hash(arg.block_hash, b)) @@ -725,8 +725,8 @@ namespace cryptonote } } - std::list<cryptonote::transaction> txs; - std::list<crypto::hash> missed; + std::vector<cryptonote::transaction> txs; + std::vector<crypto::hash> missed; if (!m_core.get_transactions(txids, txs, missed)) { LOG_ERROR_CCONTEXT("Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX, " @@ -774,10 +774,12 @@ namespace cryptonote return 1; } - for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end();) + std::vector<cryptonote::blobdata> newtxs; + newtxs.reserve(arg.txs.size()); + for (size_t i = 0; i < arg.txs.size(); ++i) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, false, true, false); + m_core.handle_incoming_tx(arg.txs[i], tvc, false, true, false); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); @@ -785,10 +787,9 @@ namespace cryptonote return 1; } if(tvc.m_should_be_relayed) - ++tx_blob_it; - else - arg.txs.erase(tx_blob_it++); + newtxs.push_back(std::move(arg.txs[i])); } + arg.txs = std::move(newtxs); if(arg.txs.size()) { @@ -996,7 +997,7 @@ skip: { const uint64_t previous_height = m_core.get_current_blockchain_height(); uint64_t start_height; - std::list<cryptonote::block_complete_entry> blocks; + std::vector<cryptonote::block_complete_entry> blocks; boost::uuids::uuid span_connection_id; if (!m_block_queue.get_next_span(start_height, blocks, span_connection_id)) { @@ -1070,7 +1071,7 @@ skip: LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()"); return 1; } - std::list<blobdata>::const_iterator it = block_entry.txs.begin(); + std::vector<blobdata>::const_iterator it = block_entry.txs.begin(); for (size_t i = 0; i < tvc.size(); ++i, ++it) { if(tvc[i].m_verifivation_failed) @@ -1167,8 +1168,20 @@ skip: + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued"; if (ELPP->vRegistry()->allowed(el::Level::Debug, "sync-info")) timing_message += std::string(": ") + m_block_queue.get_overview(); - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() + if(m_core.get_target_blockchain_height() == 0){ + MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() << timing_message); + } else { + const int completition_percent = (m_core.get_current_blockchain_height() * 100 / m_core.get_target_blockchain_height()); + if(completition_percent < 99) {//printing completion percent only if % is < of 99 cause for 99 >= this is useless + MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() + << " (" << completition_percent << "% " << (m_core.get_target_blockchain_height() - m_core.get_current_blockchain_height()) + << " blocks remaining)" << timing_message); + } else { + MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() + << timing_message); + } + } } } } @@ -1248,7 +1261,7 @@ skip: template<class t_core> bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context) const { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; std::pair<uint64_t, uint64_t> span; @@ -1267,7 +1280,7 @@ skip: // we might be in a weird case where there is a filled next span, // but it starts higher than the current height uint64_t height; - std::list<cryptonote::block_complete_entry> bcel; + std::vector<cryptonote::block_complete_entry> bcel; if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) return false; if (height > m_core.get_current_blockchain_height()) @@ -1415,7 +1428,7 @@ skip: { if (span.second == 0) { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1441,14 +1454,18 @@ skip: goto skip; } // take out blocks we already have - while (!context.m_needed_objects.empty() && m_core.have_block(context.m_needed_objects.front())) + size_t skip = 0; + while (skip < context.m_needed_objects.size() && m_core.have_block(context.m_needed_objects[skip])) { // if we're popping the last hash, record it so we can ask again from that hash, // this prevents never being able to progress on peers we get old hash lists from - if (context.m_needed_objects.size() == 1) - context.m_last_known_hash = context.m_needed_objects.front(); - context.m_needed_objects.pop_front(); + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; } + if (skip > 0) + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); @@ -1456,7 +1473,7 @@ skip: if (span.second == 0 && !force_next_span) { MDEBUG(context << " still no span reserved, we may be in the corner case of next span scheduled and everything else scheduled/filled"); - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1487,23 +1504,21 @@ skip: MERROR("ERROR: skip " << skip << ", m_needed_objects " << context.m_needed_objects.size() << ", first_context_block_height" << first_context_block_height); return false; } - while (skip--) - context.m_needed_objects.pop_front(); + if (skip > 0) + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); if (context.m_needed_objects.size() < span.second) { MERROR("ERROR: span " << span.first << "/" << span.second << ", m_needed_objects " << context.m_needed_objects.size()); return false; } - auto it = context.m_needed_objects.begin(); for (size_t n = 0; n < span.second; ++n) { - req.blocks.push_back(*it); + req.blocks.push_back(context.m_needed_objects[n]); ++count; - context.m_requested_objects.insert(*it); - auto j = it++; - context.m_needed_objects.erase(j); + context.m_requested_objects.insert(context.m_needed_objects[n]); } + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end()); } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); @@ -1664,15 +1679,10 @@ skip: { NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_arg = AUTO_VAL_INIT(fluffy_arg); fluffy_arg.current_blockchain_height = arg.current_blockchain_height; - std::list<blobdata> fluffy_txs; + std::vector<blobdata> fluffy_txs; fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; - // pre-serialize them - std::string fullBlob, fluffyBlob; - epee::serialization::store_t_to_binary(arg, fullBlob); - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - // sort peers between fluffy ones and others std::list<boost::uuids::uuid> fullConnections, fluffyConnections; m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) @@ -1694,8 +1704,18 @@ skip: }); // send fluffy ones first, we want to encourage people to run that - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + if (!fluffyConnections.empty()) + { + std::string fluffyBlob; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); + } + if (!fullConnections.empty()) + { + std::string fullBlob; + epee::serialization::store_t_to_binary(arg, fullBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + } return true; } @@ -1712,6 +1732,9 @@ skip: template<class t_core> void t_cryptonote_protocol_handler<t_core>::drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans) { + LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << + ", add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); + if (add_fail) m_p2p->add_host_fail(context.m_remote_address); @@ -1747,3 +1770,4 @@ skip: m_core.stop(); } } // namespace + diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 3b1d0d826..117790455 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -26,16 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(blocksdat "") -if(PER_BLOCK_CHECKPOINT) - if(APPLE) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - else() - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - endif() - set(blocksdat "blocksdat.o") -endif() - set(daemon_sources command_parser_executor.cpp command_server.cpp @@ -77,9 +67,7 @@ monero_private_headers(daemon monero_add_executable(daemon ${daemon_sources} ${daemon_headers} - ${daemon_private_headers} - ${blocksdat} -) + ${daemon_private_headers}) target_link_libraries(daemon PRIVATE rpc @@ -92,7 +80,6 @@ target_link_libraries(daemon daemonizer serialization daemon_rpc_server - epee ${EPEE_READLINE} version ${Boost_CHRONO_LIBRARY} @@ -103,7 +90,8 @@ target_link_libraries(daemon ${CMAKE_THREAD_LIBS_INIT} ${ZMQ_LIB} ${GNU_READLINE_LIBRARY} - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${Blocks}) set_property(TARGET daemon PROPERTY OUTPUT_NAME "monerod") diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 4673590aa..cba71bf3b 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -72,6 +72,11 @@ namespace daemon_args , "Specify maximum log file size [B]" , MAX_LOG_FILE_SIZE }; + const command_line::arg_descriptor<std::size_t> arg_max_log_files = { + "max-log-files" + , "Specify maximum number of rotated log files to be saved (no limit by setting to 0)" + , MAX_LOG_FILES + }; const command_line::arg_descriptor<std::string> arg_log_level = { "log-level" , "" diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 34d9fb4c8..1638cf505 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common/dns_utils.h" +#include "common/command_line.h" #include "version.h" #include "daemon/command_parser_executor.h" @@ -326,12 +327,26 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg if(args.size() == 4) { - ignore_battery = args[3] == "true"; + if(args[3] == "true" || command_line::is_yes(args[3]) || args[3] == "1") + { + ignore_battery = true; + } + else if(args[3] != "false" && !command_line::is_no(args[3]) && args[3] != "0") + { + return false; + } } if(args.size() >= 3) { - do_background_mining = args[2] == "true"; + if(args[2] == "true" || command_line::is_yes(args[2]) || args[2] == "1") + { + do_background_mining = true; + } + else if(args[2] != "false" && !command_line::is_no(args[2]) && args[2] != "0") + { + return false; + } } if(args.size() >= 2) @@ -599,13 +614,13 @@ bool t_command_parser_executor::print_coinbase_tx_sum(const std::vector<std::str bool t_command_parser_executor::alt_chain_info(const std::vector<std::string>& args) { - if(args.size()) + if(args.size() > 1) { - std::cout << "No parameters allowed" << std::endl; + std::cout << "usage: alt_chain_info [block_hash]" << std::endl; return false; } - return m_executor.alt_chain_info(); + return m_executor.alt_chain_info(args.size() == 1 ? args[0] : ""); } bool t_command_parser_executor::print_blockchain_dynamic_stats(const std::vector<std::string>& args) diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 144603597..35504f2c9 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -255,6 +255,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "alt_chain_info" , std::bind(&t_command_parser_executor::alt_chain_info, &m_parser, p::_1) + , "alt_chain_info [blockhash]" , "Print the information about alternative chains." ); m_command_lookup.set_handler( diff --git a/src/daemon/core.h b/src/daemon/core.h index 475f418d6..d1defd573 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -28,6 +28,7 @@ #pragma once +#include "blocks/blocks.h" #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "misc_log_ex.h" @@ -85,7 +86,12 @@ public: //initialize core here MGINFO("Initializing core..."); std::string config_subdir = get_config_subdir(); - if (!m_core.init(m_vm_HACK, config_subdir.empty() ? NULL : config_subdir.c_str())) +#if defined(PER_BLOCK_CHECKPOINT) + const cryptonote::GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData; +#else + const cryptonote::GetCheckpointsCallback& get_checkpoints = nullptr; +#endif + if (!m_core.init(m_vm_HACK, config_subdir.empty() ? NULL : config_subdir.c_str(), nullptr, get_checkpoints)) { return false; } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 48671f190..49d6d49cf 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -77,9 +77,10 @@ public: const auto testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); const auto stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + const auto regtest = command_line::get_arg(vm, cryptonote::arg_regtest_on); const auto restricted = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc); const auto main_rpc_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); - rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET, main_rpc_port, "core"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : regtest ? cryptonote::FAKECHAIN : cryptonote::MAINNET, main_rpc_port, "core"}); auto restricted_rpc_port_arg = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port; if(!command_line::is_arg_defaulted(vm, restricted_rpc_port_arg)) @@ -135,7 +136,19 @@ bool t_daemon::run(bool interactive) { throw std::runtime_error{"Can't run stopped daemon"}; } - tools::signal_handler::install(std::bind(&daemonize::t_daemon::stop_p2p, this)); + + std::atomic<bool> stop(false), shutdown(false); + boost::thread stop_thread = boost::thread([&stop, &shutdown, this] { + while (!stop) + epee::misc_utils::sleep_no_w(100); + if (shutdown) + this->stop_p2p(); + }); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + stop = true; + stop_thread.join(); + }); + tools::signal_handler::install([&stop, &shutdown](int){ stop = shutdown = true; }); try { diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 49494e889..f483ba6c9 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -84,6 +84,7 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_log_file); command_line::add_arg(core_settings, daemon_args::arg_log_level); command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); + command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); @@ -162,9 +163,10 @@ int main(int argc, char const * argv[]) const bool testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); const bool stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - if (testnet && stagenet) + const bool regtest = command_line::get_arg(vm, cryptonote::arg_regtest_on); + if (testnet + stagenet + regtest > 1) { - std::cerr << "Can't specify more than one of --tesnet and --stagenet" << ENDL; + std::cerr << "Can't specify more than one of --tesnet and --stagenet and --regtest" << ENDL; return 1; } @@ -203,7 +205,7 @@ int main(int argc, char const * argv[]) if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_file)) log_file_path = command_line::get_arg(vm, daemon_args::arg_log_file); log_file_path = bf::absolute(log_file_path, relative_path_base); - mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size)); + mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size), command_line::get_arg(vm, daemon_args::arg_max_log_files)); // Set log level if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_level)) @@ -237,11 +239,14 @@ int main(int argc, char const * argv[]) return 1; } + const char *env_rpc_login = nullptr; + const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login); + const bool use_rpc_env = !has_rpc_arg && (env_rpc_login = getenv("RPC_LOGIN")) != nullptr && strlen(env_rpc_login) > 0; boost::optional<tools::login> login{}; - if (command_line::has_arg(vm, arg.rpc_login)) + if (has_rpc_arg || use_rpc_env) { login = tools::login::parse( - command_line::get_arg(vm, arg.rpc_login), false, [](bool verify) { + has_rpc_arg ? command_line::get_arg(vm, arg.rpc_login) : std::string(env_rpc_login), false, [](bool verify) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif @@ -262,6 +267,9 @@ int main(int argc, char const * argv[]) } else { +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif std::cerr << "Unknown command: " << command.front() << std::endl; return 1; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 487853441..34d74d642 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -75,7 +75,10 @@ namespace { << "hash: " << header.hash << std::endl << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl << "POW hash: " << header.pow_hash << std::endl - << "reward: " << boost::lexical_cast<std::string>(header.reward); + << "block size: " << header.block_size << std::endl + << "block weight: " << header.block_weight << std::endl + << "num txes: " << header.num_txes << std::endl + << "reward: " << cryptonote::print_money(header.reward); } std::string get_human_time_ago(time_t t, time_t now) @@ -439,25 +442,35 @@ bool t_rpc_command_executor::show_status() { } } - tools::success_msg_writer() << boost::format("Height: %llu/%llu (%.1f%%) on %s%s, %s, net hash %s, v%u%s, %s, %u(out)+%u(in) connections, uptime %ud %uh %um %us") + std::stringstream str; + str << boost::format("Height: %llu/%llu (%.1f%%) on %s%s, %s, net hash %s, v%u%s, %s, %u(out)+%u(in) connections") % (unsigned long long)ires.height % (unsigned long long)net_height % get_sync_percentage(ires) % (ires.testnet ? "testnet" : ires.stagenet ? "stagenet" : "mainnet") % bootstrap_msg - % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed) ) : "not mining") + % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed) + std::string(" to ") + mres.address ) : "not mining") % get_mining_speed(ires.difficulty / ires.target) % (unsigned)hfres.version % get_fork_extra_info(hfres.earliest_height, net_height, ires.target) % (hfres.state == cryptonote::HardFork::Ready ? "up to date" : hfres.state == cryptonote::HardFork::UpdateNeeded ? "update needed" : "out of date, likely forked") % (unsigned)ires.outgoing_connections_count % (unsigned)ires.incoming_connections_count - % (unsigned int)floor(uptime / 60.0 / 60.0 / 24.0) - % (unsigned int)floor(fmod((uptime / 60.0 / 60.0), 24.0)) - % (unsigned int)floor(fmod((uptime / 60.0), 60.0)) - % (unsigned int)fmod(uptime, 60.0) ; + // restricted RPC does not disclose start time + if (ires.start_time) + { + str << boost::format(", uptime %ud %uh %um %us") + % (unsigned int)floor(uptime / 60.0 / 60.0 / 24.0) + % (unsigned int)floor(fmod((uptime / 60.0 / 60.0), 24.0)) + % (unsigned int)floor(fmod((uptime / 60.0), 60.0)) + % (unsigned int)fmod(uptime, 60.0) + ; + } + + tools::success_msg_writer() << str.str(); + return true; } @@ -530,6 +543,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u req.start_height = start_block_index; req.end_height = end_block_index; + req.fill_pow_hash = false; std::string fail_message = "Unsuccessful"; @@ -555,8 +569,8 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u if (!first) std::cout << std::endl; std::cout - << "height: " << header.height << ", timestamp: " << header.timestamp << ", difficulty: " << header.difficulty - << ", size: " << header.block_size << ", transactions: " << header.num_txes << std::endl + << "height: " << header.height << ", timestamp: " << header.timestamp + << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; @@ -855,8 +869,9 @@ bool t_rpc_command_executor::print_transaction_pool_long() { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << tx_info.tx_json << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -938,8 +953,9 @@ bool t_rpc_command_executor::print_transaction_pool_short() { { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -976,7 +992,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { } else { - memset(&res.pool_stats, 0, sizeof(res.pool_stats)); + res.pool_stats = {}; if (!m_rpc_server->on_get_transaction_pool_stats(req, res, false) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); @@ -994,7 +1010,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0; std::string backlog_message; - const uint64_t full_reward_zone = ires.block_size_limit / 2; + const uint64_t full_reward_zone = ires.block_weight_limit / 2; if (res.pool_stats.bytes_total <= full_reward_zone) { backlog_message = "no backlog"; @@ -1628,7 +1644,7 @@ bool t_rpc_command_executor::print_coinbase_tx_sum(uint64_t height, uint64_t cou return true; } -bool t_rpc_command_executor::alt_chain_info() +bool t_rpc_command_executor::alt_chain_info(const std::string &tip) { cryptonote::COMMAND_RPC_GET_INFO::request ireq; cryptonote::COMMAND_RPC_GET_INFO::response ires; @@ -1663,12 +1679,32 @@ bool t_rpc_command_executor::alt_chain_info() } } - tools::msg_writer() << boost::lexical_cast<std::string>(res.chains.size()) << " alternate chains found:"; - for (const auto &chain: res.chains) + if (tip.empty()) + { + tools::msg_writer() << boost::lexical_cast<std::string>(res.chains.size()) << " alternate chains found:"; + for (const auto &chain: res.chains) + { + uint64_t start_height = (chain.height - chain.length + 1); + tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) + << " deep), diff " << chain.difficulty << ": " << chain.block_hash; + } + } + else { - uint64_t start_height = (chain.height - chain.length + 1); - tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) - << " deep), diff " << chain.difficulty << ": " << chain.block_hash; + const auto i = std::find_if(res.chains.begin(), res.chains.end(), [&tip](cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info &info){ return info.block_hash == tip; }); + if (i != res.chains.end()) + { + const auto &chain = *i; + tools::success_msg_writer() << "Found alternate chain with tip " << tip; + uint64_t start_height = (chain.height - chain.length + 1); + tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) + << " deep), diff " << chain.difficulty << ":"; + for (const std::string &block_id: chain.block_hashes) + tools::msg_writer() << " " << block_id; + tools::msg_writer() << "Chain parent on main chain: " << chain.main_chain_parent_block; + } + else + tools::fail_msg_writer() << "Block hash " << tip << " is not the tip of any known alternate chain"; } return true; } @@ -1679,13 +1715,16 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) cryptonote::COMMAND_RPC_GET_INFO::response ires; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request bhreq; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response bhres; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request fereq; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request fereq; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_HARD_FORK_INFO::request hfreq; + cryptonote::COMMAND_RPC_HARD_FORK_INFO::response hfres; epee::json_rpc::error error_resp; std::string fail_message = "Problem fetching info"; fereq.grace_blocks = 0; + hfreq.version = HF_VERSION_PER_BYTE_FEE; if (m_is_rpc) { if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str())) @@ -1696,6 +1735,10 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) { return true; } + if (!m_rpc_client->json_rpc_request(hfreq, hfres, "hard_fork_info", fail_message.c_str())) + { + return true; + } } else { @@ -1704,15 +1747,20 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) tools::fail_msg_writer() << make_error(fail_message, ires.status); return true; } - if (!m_rpc_server->on_get_per_kb_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_base_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, feres.status); return true; } + if (!m_rpc_server->on_hard_fork_info(hfreq, hfres, error_resp) || hfres.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, hfres.status); + return true; + } } tools::msg_writer() << "Height: " << ires.height << ", diff " << ires.difficulty << ", cum. diff " << ires.cumulative_difficulty - << ", target " << ires.target << " sec" << ", dyn fee " << cryptonote::print_money(feres.fee) << "/kB"; + << ", target " << ires.target << " sec" << ", dyn fee " << cryptonote::print_money(feres.fee) << "/" << (hfres.enabled ? "byte" : "kB"); if (nblocks > 0) { @@ -1721,6 +1769,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) bhreq.start_height = ires.height - nblocks; bhreq.end_height = ires.height - 1; + bhreq.fill_pow_hash = false; if (m_is_rpc) { if (!m_rpc_client->json_rpc_request(bhreq, bhres, "getblockheadersrange", fail_message.c_str())) @@ -1740,8 +1789,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) double avgdiff = 0; double avgnumtxes = 0; double avgreward = 0; - std::vector<uint64_t> sizes; - sizes.reserve(nblocks); + std::vector<uint64_t> weights; + weights.reserve(nblocks); uint64_t earliest = std::numeric_limits<uint64_t>::max(), latest = 0; std::vector<unsigned> major_versions(256, 0), minor_versions(256, 0); for (const auto &bhr: bhres.headers) @@ -1749,7 +1798,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff += bhr.difficulty; avgnumtxes += bhr.num_txes; avgreward += bhr.reward; - sizes.push_back(bhr.block_size); + weights.push_back(bhr.block_weight); static_assert(sizeof(bhr.major_version) == 1, "major_version expected to be uint8_t"); static_assert(sizeof(bhr.minor_version) == 1, "major_version expected to be uint8_t"); major_versions[(unsigned)bhr.major_version]++; @@ -1760,9 +1809,9 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff /= nblocks; avgnumtxes /= nblocks; avgreward /= nblocks; - uint64_t median_block_size = epee::misc_utils::median(sizes); + uint64_t median_block_weight = epee::misc_utils::median(weights); tools::msg_writer() << "Last " << nblocks << ": avg. diff " << (uint64_t)avgdiff << ", " << (latest - earliest) / nblocks << " avg sec/block, avg num txes " << avgnumtxes - << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block size " << median_block_size; + << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block weight " << median_block_weight; unsigned int max_major = 256, max_minor = 256; while (max_major > 0 && !major_versions[--max_major]); @@ -1895,7 +1944,7 @@ bool t_rpc_command_executor::sync_info() for (const auto &s: res.spans) if (s.rate > 0.0f && s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << epee::string_tools::pad_string(p.info.state, 16) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 46168c93b..9e6010c5b 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -143,7 +143,7 @@ public: bool print_coinbase_tx_sum(uint64_t height, uint64_t count); - bool alt_chain_info(); + bool alt_chain_info(const std::string &tip); bool print_blockchain_dynamic_stats(uint64_t nblocks); diff --git a/src/daemonizer/windows_daemonizer.inl b/src/daemonizer/windows_daemonizer.inl index 8077f29fb..7e61e3603 100644 --- a/src/daemonizer/windows_daemonizer.inl +++ b/src/daemonizer/windows_daemonizer.inl @@ -31,6 +31,7 @@ #include "common/util.h" #include "daemonizer/windows_service.h" #include "daemonizer/windows_service_runner.h" +#include "cryptonote_core/cryptonote_core.h" #include <shlobj.h> #include <boost/filesystem/operations.hpp> diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index dfbd3b864..3e2552230 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -27,6 +27,8 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/filesystem.hpp> +#include <boost/algorithm/string/join.hpp> +#include <boost/range/adaptor/transformed.hpp> #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/tx_extra.h" #include "cryptonote_core/blockchain.h" @@ -41,6 +43,15 @@ using namespace epee; using namespace cryptonote; +static std::string extra_nonce_to_string(const cryptonote::tx_extra_nonce &extra_nonce) +{ + if (extra_nonce.nonce.size() == 9 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID) + return "encrypted payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + if (extra_nonce.nonce.size() == 33 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_PAYMENT_ID) + return "plaintext payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + return epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce); +} + static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fields) { std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; @@ -49,8 +60,9 @@ static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fi std::cout << "field " << n << ": "; if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get<cryptonote::tx_extra_padding>(fields[n]).size << " bytes"; else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key; - else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << extra_nonce_to_string(boost::get<cryptonote::tx_extra_nonce>(fields[n])); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root; + else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get<cryptonote::tx_extra_additional_pub_keys>(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_mysterious_minergate>(fields[n]).data); else std::cout << "unknown"; std::cout << std::endl; @@ -165,9 +177,14 @@ int main(int argc, char* argv[]) std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; } - else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { - std::cout << "Parsed transaction:" << std::endl; +/* + if (tx.pruned) + std::cout << "Parsed pruned transaction:" << std::endl; + else +*/ + std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; bool parsed = cryptonote::parse_tx_extra(tx.extra, fields); diff --git a/src/debug_utilities/object_sizes.cpp b/src/debug_utilities/object_sizes.cpp index a3d037a59..2281d0734 100644 --- a/src/debug_utilities/object_sizes.cpp +++ b/src/debug_utilities/object_sizes.cpp @@ -29,6 +29,7 @@ #include <map> #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/tx_extra.h" +#include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "p2p/p2p_protocol_defs.h" #include "net/connection_basic.hpp" @@ -94,6 +95,11 @@ int main(int argc, char* argv[]) SL(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core>>); SL(nodetool::p2p_connection_context_t<cryptonote::t_cryptonote_protocol_handler<cryptonote::core>::connection_context>); SL(nodetool::network_address_old); + SL(nodetool::peerlist_entry_base<nodetool::network_address_old>); + + SL(nodetool::network_config); + SL(nodetool::basic_node_data); + SL(cryptonote::CORE_SYNC_DATA); SL(tools::wallet2::transfer_details); SL(tools::wallet2::payment_details); diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index cc2d20a54..91d670b73 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -32,29 +32,33 @@ set(device_sources log.cpp ) -if(PCSC_FOUND) - set(device_sources ${device_sources} device_ledger.cpp) +if(HIDAPI_FOUND) + set(device_sources + ${device_sources} + device_ledger.cpp + device_io_hid.cpp + ) endif() set(device_headers device.hpp + device_io.hpp device_default.hpp + device_cold.hpp log.hpp ) -if(PCSC_FOUND) - set(device_headers ${device_headers} device_ledger.hpp) +if(HIDAPI_FOUND) + set(device_headers + ${device_headers} + device_ledger.hpp + device_io_hid.hpp + ) endif() set(device_private_headers) -if(PER_BLOCK_CHECKPOINT) - set(Blocks "blocks") -else() - set(Blocks "") -endif() - monero_private_headers(device ${device_private_headers}) @@ -65,10 +69,10 @@ monero_add_library(device target_link_libraries(device PUBLIC - ${PCSC_LIBRARIES} + ${HIDAPI_LIBRARIES} cncrypto ringct_basic ${OPENSSL_CRYPTO_LIBRARIES} + ${Boost_SERIALIZATION_LIBRARY} PRIVATE - ${Blocks} ${EXTRA_LIBRARIES}) diff --git a/src/device/device.cpp b/src/device/device.cpp index 983f59b60..d5e3031ff 100644 --- a/src/device/device.cpp +++ b/src/device/device.cpp @@ -29,7 +29,7 @@ #include "device.hpp" #include "device_default.hpp" -#ifdef HAVE_PCSC +#ifdef WITH_DEVICE_LEDGER #include "device_ledger.hpp" #endif #include "misc_log_ex.h" @@ -39,32 +39,67 @@ namespace hw { /* ======================================================================= */ /* SETUP */ - /* ======================================================================= */ - device& get_device(const std::string device_descriptor) { - - struct s_devices { - std::map<std::string, std::unique_ptr<device>> registry; - s_devices() : registry() { - hw::core::register_all(registry); - #ifdef HAVE_PCSC - hw::ledger::register_all(registry); - #endif - }; - }; - - static const s_devices devices; + /* ======================================================================= */ + + static device_registry *get_device_registry(bool clear = false){ + static device_registry *registry = new device_registry(); + if (clear) + { + delete registry; + registry = NULL; + } + return registry; + } + + static void clear_device_registry(){ + get_device_registry(true); + } + + device_registry::device_registry(){ + hw::core::register_all(registry); + #ifdef WITH_DEVICE_LEDGER + hw::ledger::register_all(registry); + #endif + atexit(clear_device_registry); + } + + bool device_registry::register_device(const std::string & device_name, device * hw_device){ + auto search = registry.find(device_name); + if (search != registry.end()){ + return false; + } - auto device = devices.registry.find(device_descriptor); - if (device == devices.registry.end()) { - MERROR("device not found in registry: '" << device_descriptor << "'\n" << - "known devices:"); - - for( const auto& sm_pair : devices.registry ) { + registry.insert(std::make_pair(device_name, std::unique_ptr<device>(hw_device))); + return true; + } + + device& device_registry::get_device(const std::string & device_descriptor){ + // Device descriptor can contain further specs after first : + auto delim = device_descriptor.find(':'); + auto device_descriptor_lookup = device_descriptor; + if (delim != std::string::npos) { + device_descriptor_lookup = device_descriptor.substr(0, delim); + } + + auto device = registry.find(device_descriptor_lookup); + if (device == registry.end()) { + MERROR("Device not found in registry: '" << device_descriptor << "'. Known devices: "); + for( const auto& sm_pair : registry ) { MERROR(" - " << sm_pair.first); } - throw std::runtime_error("device not found: "+ device_descriptor); + throw std::runtime_error("device not found: " + device_descriptor); } return *device->second; } + device& get_device(const std::string & device_descriptor) { + device_registry *registry = get_device_registry(); + return registry->get_device(device_descriptor); + } + + bool register_device(const std::string & device_name, device * hw_device){ + device_registry *registry = get_device_registry(); + return registry->register_device(device_name, hw_device); + } + } diff --git a/src/device/device.hpp b/src/device/device.hpp index 9df0cb39d..dd9ad4332 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -47,12 +47,14 @@ #include "crypto/crypto.h" #include "crypto/chacha.h" #include "ringct/rctTypes.h" +#include "cryptonote_config.h" + #ifndef USE_DEVICE_LEDGER #define USE_DEVICE_LEDGER 1 #endif -#if !defined(HAVE_PCSC) +#if !defined(HAVE_HIDAPI) #undef USE_DEVICE_LEDGER #define USE_DEVICE_LEDGER 0 #endif @@ -78,14 +80,13 @@ namespace hw { return false; } - class device { protected: std::string name; public: - device() {} + device(): mode(NONE) {} device(const device &hwdev) {} virtual ~device() {} @@ -96,6 +97,19 @@ namespace hw { TRANSACTION_CREATE_FAKE, TRANSACTION_PARSE }; + enum device_type + { + SOFTWARE = 0, + LEDGER = 1, + TREZOR = 2 + }; + + + enum device_protocol_t { + PROTOCOL_DEFAULT, + PROTOCOL_PROXY, // Originally defined by Ledger + PROTOCOL_COLD, // Originally defined by Trezor + }; /* ======================================================================= */ /* SETUP/TEARDOWN */ @@ -109,8 +123,12 @@ namespace hw { virtual bool connect(void) = 0; virtual bool disconnect(void) = 0; - virtual bool set_mode(device_mode mode) = 0; + virtual bool set_mode(device_mode mode) { this->mode = mode; return true; } + virtual device_mode get_mode() const { return mode; } + + virtual device_type get_type() const = 0; + virtual device_protocol_t device_protocol() const { return PROTOCOL_DEFAULT; }; /* ======================================================================= */ /* LOCKER */ @@ -125,7 +143,7 @@ namespace hw { /* ======================================================================= */ virtual bool get_public_address(cryptonote::account_public_address &pubkey) = 0; virtual bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) = 0; - virtual bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) = 0; + virtual bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) = 0; /* ======================================================================= */ /* SUB ADDRESS */ @@ -194,6 +212,14 @@ namespace hw { virtual bool mlsag_sign(const rct::key &c, const rct::keyV &xx, const rct::keyV &alpha, const size_t rows, const size_t dsRows, rct::keyV &ss) = 0; virtual bool close_tx(void) = 0; + + virtual bool has_ki_cold_sync(void) const { return false; } + virtual bool has_tx_cold_sign(void) const { return false; } + + virtual void set_network_type(cryptonote::network_type network_type) { } + + protected: + device_mode mode; } ; struct reset_mode { @@ -202,6 +228,17 @@ namespace hw { ~reset_mode() { hwref.set_mode(hw::device::NONE);} }; - device& get_device(const std::string device_descriptor) ; + class device_registry { + private: + std::map<std::string, std::unique_ptr<device>> registry; + + public: + device_registry(); + bool register_device(const std::string & device_name, device * hw_device); + device& get_device(const std::string & device_descriptor); + }; + + device& get_device(const std::string & device_descriptor); + bool register_device(const std::string & device_name, device * hw_device); } diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp new file mode 100644 index 000000000..22128cec1 --- /dev/null +++ b/src/device/device_cold.hpp @@ -0,0 +1,71 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_COLD_H +#define MONERO_DEVICE_COLD_H + +#include "wallet/wallet2.h" +#include <boost/function.hpp> + + +namespace hw { + + typedef struct wallet_shim { + boost::function<crypto::public_key (const tools::wallet2::transfer_details &td)> get_tx_pub_key_from_received_outs; + } wallet_shim; + + class tx_aux_data { + public: + std::vector<std::string> tx_device_aux; // device generated aux data + std::vector<cryptonote::address_parse_info> tx_recipients; // as entered by user + }; + + class device_cold { + public: + + using exported_key_image = std::vector<std::pair<crypto::key_image, crypto::signature>>; + + /** + * Key image sync with the cold protocol. + */ + virtual void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + exported_key_image & ski) =0; + + /** + * Signs unsigned transaction with the cold protocol. + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + tx_aux_data & aux_data) =0; + }; +} + +#endif //MONERO_DEVICE_COLD_H diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 0071f7d4f..1e3d80949 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -69,21 +69,21 @@ namespace hw { } bool device_default::init(void) { - dfns(); + return true; } bool device_default::release() { - dfns(); + return true; } bool device_default::connect(void) { - dfns(); + return true; } bool device_default::disconnect() { - dfns(); + return true; } bool device_default::set_mode(device_mode mode) { - return true; + return device::set_mode(mode); } /* ======================================================================= */ @@ -100,14 +100,14 @@ namespace hw { /* WALLET & ADDRESS */ /* ======================================================================= */ - bool device_default::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) { + bool device_default::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) { const crypto::secret_key &view_key = keys.m_view_secret_key; const crypto::secret_key &spend_key = keys.m_spend_secret_key; - tools::scrubbed_arr<char, sizeof(view_key) + sizeof(spend_key) + 1> data; + epee::mlocked<tools::scrubbed_arr<char, sizeof(view_key) + sizeof(spend_key) + 1>> data; memcpy(data.data(), &view_key, sizeof(view_key)); memcpy(data.data() + sizeof(view_key), &spend_key, sizeof(spend_key)); data[sizeof(data) - 1] = CHACHA8_KEY_TAIL; - crypto::generate_chacha_key(data.data(), sizeof(data), key); + crypto::generate_chacha_key(data.data(), sizeof(data), key, kdf_rounds); return true; } bool device_default::get_public_address(cryptonote::account_public_address &pubkey) { diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 771fbba72..5c59a9066 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -61,6 +61,8 @@ namespace hw { bool set_mode(device_mode mode) override; + device_type get_type() const override {return device_type::SOFTWARE;}; + /* ======================================================================= */ /* LOCKER */ /* ======================================================================= */ @@ -73,7 +75,7 @@ namespace hw { /* ======================================================================= */ bool get_public_address(cryptonote::account_public_address &pubkey) override; bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; - bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) override; + bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) override; /* ======================================================================= */ /* SUB ADDRESS */ diff --git a/src/device/device_io.hpp b/src/device/device_io.hpp new file mode 100644 index 000000000..96163a211 --- /dev/null +++ b/src/device/device_io.hpp @@ -0,0 +1,56 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + + + + +#pragma once + + +namespace hw { + namespace io { + + class device_io { + + public: + + device_io() {}; + ~device_io() {}; + + virtual void init() = 0; + virtual void release() = 0; + + virtual void connect(void *parms) = 0; + virtual void disconnect() = 0; + virtual bool connected() const = 0; + + virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) = 0; + }; + }; +}; diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp new file mode 100644 index 000000000..1aadfb9ea --- /dev/null +++ b/src/device/device_io_hid.cpp @@ -0,0 +1,331 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +#if defined(HAVE_HIDAPI) + +#include <boost/scope_exit.hpp> +#include "log.hpp" +#include "device_io_hid.hpp" + +namespace hw { + namespace io { + + #undef MONERO_DEFAULT_LOG_CATEGORY + #define MONERO_DEFAULT_LOG_CATEGORY "device.io" + + #define ASSERT_X(exp,msg) CHECK_AND_ASSERT_THROW_MES(exp, msg); + + #define MAX_BLOCK 64 + + static std::string safe_hid_error(hid_device *hwdev) { + if (hwdev) { + return std::string((char*)hid_error(hwdev)); + } + return std::string("NULL device"); + } + + static std::string safe_hid_path(const hid_device_info *hwdev_info) { + if (hwdev_info && hwdev_info->path) { + return std::string(hwdev_info->path); + } + return std::string("NULL path"); + } + + device_io_hid::device_io_hid(unsigned short c, unsigned char t, unsigned int ps, unsigned int to) : + channel(c), + tag(t), + packet_size(ps), + timeout(to), + usb_vid(0), + usb_pid(0), + usb_device(NULL) { + } + + device_io_hid::device_io_hid() : device_io_hid(DEFAULT_CHANNEL, DEFAULT_TAG, DEFAULT_PACKET_SIZE, DEFAULT_TIMEOUT) { + } + + void device_io_hid::io_hid_log(int read, unsigned char* buffer, int block_len) { + if (hid_verbose) { + char strbuffer[1024]; + hw::buffer_to_str(strbuffer, sizeof(strbuffer), (char*)buffer, block_len); + MDEBUG( "HID " << (read?"<":">") <<" : "<<strbuffer); + } + } + + void device_io_hid::init() { + int r; + r = hid_init(); + ASSERT_X(r>=0, "Unable to init hidapi library. Error "+std::to_string(r)+": "+safe_hid_error(this->usb_device)); + } + + void device_io_hid::connect(void *params) { + hid_conn_params *p = (struct hid_conn_params*)params; + this->connect(p->vid, p->pid, p->interface_number, p->usage_page); + } + + hid_device_info *device_io_hid::find_device(hid_device_info *devices_list, boost::optional<int> interface_number, boost::optional<unsigned short> usage_page) { + bool select_any = !interface_number && !usage_page; + + MDEBUG( "Looking for " << + (select_any ? "any HID Device" : "HID Device with") << + (interface_number ? (" interface_number " + std::to_string(interface_number.value())) : "") << + ((interface_number && usage_page) ? " or" : "") << + (usage_page ? (" usage_page " + std::to_string(usage_page.value())) : "")); + + hid_device_info *result = nullptr; + for (; devices_list != nullptr; devices_list = devices_list->next) { + BOOST_SCOPE_EXIT(&devices_list, &result) { + MDEBUG( (result == devices_list ? "SELECTED" : "SKIPPED ") << + " HID Device" << + " path " << safe_hid_path(devices_list) << + " interface_number " << devices_list->interface_number << + " usage_page " << devices_list->usage_page); + } + BOOST_SCOPE_EXIT_END + + if (result != nullptr) { + continue; + } + + if (select_any) { + result = devices_list; + } else if (interface_number && devices_list->interface_number == interface_number.value()) { + result = devices_list; + } else if (usage_page && devices_list->usage_page == usage_page.value()) { + result = devices_list; + } + } + + return result; + } + + void device_io_hid::connect(unsigned int vid, unsigned int pid, boost::optional<int> interface_number, boost::optional<unsigned short> usage_page) { + hid_device_info *hwdev_info_list; + hid_device *hwdev; + + this->disconnect(); + + hwdev_info_list = hid_enumerate(vid, pid); + ASSERT_X(hwdev_info_list, "Unable to enumerate device "+std::to_string(vid)+":"+std::to_string(vid)+ ": "+ safe_hid_error(this->usb_device)); + hwdev = NULL; + if (hid_device_info *device = find_device(hwdev_info_list, interface_number, usage_page)) { + hwdev = hid_open_path(device->path); + } + hid_free_enumeration(hwdev_info_list); + ASSERT_X(hwdev, "Unable to open device "+std::to_string(pid)+":"+std::to_string(vid)); + this->usb_vid = vid; + this->usb_pid = pid; + this->usb_device = hwdev; + } + + + bool device_io_hid::connected() const { + return this->usb_device != NULL; + } + + int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) { + unsigned char buffer[400]; + unsigned char padding_buffer[MAX_BLOCK+1]; + unsigned int result; + int hid_ret; + unsigned int sw_offset; + unsigned int remaining; + unsigned int offset = 0; + + ASSERT_X(this->usb_device,"No device opened"); + + //Split command in several HID packet + memset(buffer, 0, sizeof(buffer)); + result = this->wrapCommand(command, cmd_len, buffer, sizeof(buffer)); + remaining = result; + + while (remaining > 0) { + int block_size = (remaining > MAX_BLOCK ? MAX_BLOCK : remaining); + memset(padding_buffer, 0, sizeof(padding_buffer)); + memcpy(padding_buffer+1, buffer + offset, block_size); + io_hid_log(0, padding_buffer, block_size+1); + hid_ret = hid_write(this->usb_device, padding_buffer, block_size+1); + ASSERT_X(hid_ret>=0, "Unable to send hidapi command. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); + offset += block_size; + remaining -= block_size; + } + + //get first response + memset(buffer, 0, sizeof(buffer)); + hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + ASSERT_X(hid_ret>=0, "Unable to read hidapi response. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); + result = (unsigned int)hid_ret; + io_hid_log(1, buffer, result); + offset = MAX_BLOCK; + //parse first response and get others if any + for (;;) { + result = this->unwrapReponse(buffer, offset, response, max_resp_len); + if (result != 0) { + break; + } + hid_ret = hid_read_timeout(this->usb_device, buffer + offset, MAX_BLOCK, this->timeout); + ASSERT_X(hid_ret>=0, "Unable to receive hidapi response. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); + result = (unsigned int)hid_ret; + io_hid_log(1, buffer + offset, result); + offset += MAX_BLOCK; + } + return result; + } + + void device_io_hid::disconnect(void) { + if (this->usb_device) { + hid_close(this->usb_device); + } + this->usb_vid = 0; + this->usb_pid = 0; + this->usb_device = NULL; + } + + void device_io_hid::release() { + /* Do not exit, as the lib context is global*/ + //hid_exit(); + } + + unsigned int device_io_hid::wrapCommand(const unsigned char *command, size_t command_len, unsigned char *out, size_t out_len) { + unsigned int sequence_idx = 0; + unsigned int offset = 0; + unsigned int offset_out = 0; + unsigned int block_size; + + ASSERT_X(this->packet_size >= 3, "Invalid Packet size: "+std::to_string(this->packet_size)) ; + ASSERT_X(out_len >= 7, "out_len too short: "+std::to_string(out_len)); + + out_len -= 7; + out[offset_out++] = ((this->channel >> 8) & 0xff); + out[offset_out++] = (this->channel & 0xff); + out[offset_out++] = this->tag; + out[offset_out++] = ((sequence_idx >> 8) & 0xff); + out[offset_out++] = (sequence_idx & 0xff); + sequence_idx++; + out[offset_out++] = ((command_len >> 8) & 0xff); + out[offset_out++] = (command_len & 0xff); + block_size = (command_len > this->packet_size - 7 ? this->packet_size - 7 : command_len); + ASSERT_X(out_len >= block_size, "out_len too short: "+std::to_string(out_len)); + out_len -= block_size; + memcpy(out + offset_out, command + offset, block_size); + offset_out += block_size; + offset += block_size; + while (offset != command_len) { + ASSERT_X(out_len >= 5, "out_len too short: "+std::to_string(out_len)); + out_len -= 5; + out[offset_out++] = ((this->channel >> 8) & 0xff); + out[offset_out++] = (this->channel & 0xff); + out[offset_out++] = this->tag; + out[offset_out++] = ((sequence_idx >> 8) & 0xff); + out[offset_out++] = (sequence_idx & 0xff); + sequence_idx++; + block_size = ((command_len - offset) > this->packet_size - 5 ? this->packet_size - 5 : command_len - offset); + ASSERT_X(out_len >= block_size, "out_len too short: "+std::to_string(out_len)); + out_len -= block_size; + memcpy(out + offset_out, command + offset, block_size); + offset_out += block_size; + offset += block_size; + } + while ((offset_out % this->packet_size) != 0) { + ASSERT_X(out_len >= 1, "out_len too short: "+std::to_string(out_len)); + out_len--; + out[offset_out++] = 0; + } + return offset_out; + } + + /* + * return 0 if more data are needed + * >0 if response is fully available + */ + unsigned int device_io_hid::unwrapReponse(const unsigned char *data, size_t data_len, unsigned char *out, size_t out_len) { + unsigned int sequence_idx = 0; + unsigned int offset = 0; + unsigned int offset_out = 0; + unsigned int response_len; + unsigned int block_size; + unsigned int val; + + //end? + if ((data == NULL) || (data_len < 7 + 5)) { + return 0; + } + + //check hid header + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == this->channel, "Wrong Channel"); + val = data[offset]; + offset++; + ASSERT_X(val == this->tag, "Wrong TAG"); + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == sequence_idx, "Wrong sequence_idx"); + + //fetch + response_len = (data[offset++] << 8); + response_len |= data[offset++]; + ASSERT_X(out_len >= response_len, "Out Buffer too short"); + if (data_len < (7 + response_len)) { + return 0; + } + block_size = (response_len > (this->packet_size - 7) ? this->packet_size - 7 : response_len); + memcpy(out + offset_out, data + offset, block_size); + offset += block_size; + offset_out += block_size; + while (offset_out != response_len) { + sequence_idx++; + if (offset == data_len) { + return 0; + } + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == this->channel, "Wrong Channel"); + val = data[offset]; + offset++; + ASSERT_X(val == this->tag, "Wrong TAG"); + val = (data[offset]<<8) + data[offset+1]; + offset += 2; + ASSERT_X(val == sequence_idx, "Wrong sequence_idx"); + + block_size = ((response_len - offset_out) > this->packet_size - 5 ? this->packet_size - 5 : response_len - offset_out); + if (block_size > (data_len - offset)) { + return 0; + } + memcpy(out + offset_out, data + offset, block_size); + offset += block_size; + offset_out += block_size; + } + return offset_out; + } + + + } +} + +#endif //#if defined(HAVE_HIDAPI) diff --git a/src/device/device_io_hid.hpp b/src/device/device_io_hid.hpp new file mode 100644 index 000000000..bb0f0a814 --- /dev/null +++ b/src/device/device_io_hid.hpp @@ -0,0 +1,110 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#if defined(HAVE_HIDAPI) + +#include <boost/optional/optional.hpp> +#include <hidapi/hidapi.h> +#include "device_io.hpp" + +#pragma once + +namespace hw { + namespace io { + + + + /** HID class base. Commands are formated as follow: + * + * |----------------------------------------------------------| + * | 2 bytes | 1 byte | 2 bytes | 2 bytes | len bytes | + * |-----------|----------|-----------|----------|------------| + * | channel | tag | sequence | len | payload | + * |----------------------------------------------------------| + */ + + + struct hid_conn_params { + unsigned int vid; + unsigned int pid; + int interface_number; + unsigned short usage_page; + }; + + + class device_io_hid: device_io { + + + private: + + + unsigned short channel; + unsigned char tag; + unsigned int packet_size; + unsigned int timeout; + + unsigned int usb_vid; + unsigned int usb_pid; + hid_device *usb_device; + + void io_hid_log(int read, unsigned char* buf, int buf_len); + void io_hid_init(); + void io_hid_exit() ; + void io_hid_open(int vid, int pid, int mode); + void io_hid_close (void); + + unsigned int wrapCommand(const unsigned char *command, size_t command_len, unsigned char *out, size_t out_len); + unsigned int unwrapReponse(const unsigned char *data, size_t data_len, unsigned char *out, size_t out_len); + + hid_device_info *find_device(hid_device_info *devices_list, boost::optional<int> interface_number, boost::optional<unsigned short> usage_page); + + public: + bool hid_verbose = false; + + static const unsigned short DEFAULT_CHANNEL = 0x0001; + static const unsigned char DEFAULT_TAG = 0x01; + static const unsigned int DEFAULT_PACKET_SIZE = 64; + static const unsigned int DEFAULT_TIMEOUT = 120000; + + device_io_hid(unsigned short channel, unsigned char tag, unsigned int packet_zize, unsigned int timeout); + device_io_hid(); + ~device_io_hid() {}; + + void init(); + void connect(void *params); + void connect(unsigned int vid, unsigned int pid, boost::optional<int> interface_number, boost::optional<unsigned short> usage_page); + bool connected() const; + int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len); + void disconnect(); + void release(); + }; + }; +}; + +#endif //#if defined(HAVE_HIDAPI) diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 3b9ab6744..0a86e6987 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -48,12 +48,12 @@ namespace hw { /* ===================================================================== */ /* === Debug ==== */ /* ===================================================================== */ + void set_apdu_verbose(bool verbose) { apdu_verbose = verbose; } #define TRACKD MTRACE("hw") - #define ASSERT_RV(rv) CHECK_AND_ASSERT_THROW_MES((rv)==SCARD_S_SUCCESS, "Fail SCard API : (" << (rv) << ") "<< pcsc_stringify_error(rv)<<" Device="<<this->id<<", hCard="<<hCard<<", hContext="<<hContext); #define ASSERT_SW(sw,ok,msk) CHECK_AND_ASSERT_THROW_MES(((sw)&(mask))==(ok), "Wrong Device Status : SW=" << std::hex << (sw) << " (EXPECT=" << std::hex << (ok) << ", MASK=" << std::hex << (mask) << ")") ; #define ASSERT_T0(exp) CHECK_AND_ASSERT_THROW_MES(exp, "Protocol assert failure: "#exp ) ; #define ASSERT_X(exp,msg) CHECK_AND_ASSERT_THROW_MES(exp, msg); @@ -127,7 +127,8 @@ namespace hw { } bool operator==(const crypto::key_derivation &d0, const crypto::key_derivation &d1) { - return !memcmp(&d0, &d1, sizeof(d0)); + static_assert(sizeof(crypto::key_derivation) == 32, "key_derivation must be 32 bytes"); + return !crypto_verify_32((const unsigned char*)&d0, (const unsigned char*)&d1); } /* ===================================================================== */ @@ -175,39 +176,8 @@ namespace hw { #define INS_GET_RESPONSE 0xc0 - void device_ledger::logCMD() { - if (apdu_verbose) { - char strbuffer[1024]; - sprintf(strbuffer, "%.02x %.02x %.02x %.02x %.02x ", - this->buffer_send[0], - this->buffer_send[1], - this->buffer_send[2], - this->buffer_send[3], - this->buffer_send[4] - ); - buffer_to_str(strbuffer+strlen(strbuffer), sizeof(strbuffer), (char*)(this->buffer_send+5), this->length_send-5); - MDEBUG( "CMD :" << strbuffer); - } - } - - void device_ledger::logRESP() { - if (apdu_verbose) { - char strbuffer[1024]; - sprintf(strbuffer, "%.02x%.02x ", - this->buffer_recv[this->length_recv-2], - this->buffer_recv[this->length_recv-1] - ); - buffer_to_str(strbuffer+strlen(strbuffer), sizeof(strbuffer), (char*)(this->buffer_recv), this->length_recv-2); - MDEBUG( "RESP :" << strbuffer); - - } - } - - /* -------------------------------------------------------------- */ - device_ledger::device_ledger() { + device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 120000) { this->id = device_id++; - this->hCard = 0; - this->hContext = 0; this->reset_buffer(); this->mode = NONE; this->has_view_key = false; @@ -260,45 +230,79 @@ namespace hw { MDEBUG( "Device "<<this->name << " UNLOCKed"); } + /* ======================================================================= */ - /* MISC */ + /* IO */ /* ======================================================================= */ - bool device_ledger::reset() { - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_RESET; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + void device_ledger::logCMD() { + if (apdu_verbose) { + char strbuffer[1024]; + snprintf(strbuffer, sizeof(strbuffer), "%.02x %.02x %.02x %.02x %.02x ", + this->buffer_send[0], + this->buffer_send[1], + this->buffer_send[2], + this->buffer_send[3], + this->buffer_send[4] + ); + const size_t len = strlen(strbuffer); + buffer_to_str(strbuffer+len, sizeof(strbuffer)-len, (char*)(this->buffer_send+5), this->length_send-5); + MDEBUG( "CMD : " << strbuffer); + } + } + + void device_ledger::logRESP() { + if (apdu_verbose) { + char strbuffer[1024]; + snprintf(strbuffer, sizeof(strbuffer), "%.04x ", this->sw); + const size_t len = strlen(strbuffer); + buffer_to_str(strbuffer+len, sizeof(strbuffer)-len, (char*)(this->buffer_recv), this->length_recv); + MDEBUG( "RESP : " << strbuffer); + + } + } + + int device_ledger::set_command_header(unsigned char ins, unsigned char p1, unsigned char p2) { + reset_buffer(); + int offset = 0; + this->buffer_send[0] = 0x00; + this->buffer_send[1] = ins; + this->buffer_send[2] = p1; + this->buffer_send[3] = p2; + this->buffer_send[4] = 0x00; + return 5; + } + + int device_ledger::set_command_header_noopt(unsigned char ins, unsigned char p1, unsigned char p2) { + int offset = set_command_header(ins, p1, p2); + //options + this->buffer_send[offset++] = 0; + this->buffer_send[4] = offset - 5; + return offset; + } + + void device_ledger::send_simple(unsigned char ins, unsigned char p1) { + this->length_send = set_command_header_noopt(ins, p1); + this->exchange(); + } + + bool device_ledger::reset() { + send_simple(INS_RESET); return true; } unsigned int device_ledger::exchange(unsigned int ok, unsigned int mask) { - LONG rv; - int sw; - - ASSERT_T0(this->length_send <= BUFFER_SEND_SIZE); logCMD(); - this->length_recv = BUFFER_RECV_SIZE; - rv = SCardTransmit(this->hCard, - SCARD_PCI_T0, this->buffer_send, this->length_send, - NULL, this->buffer_recv, &this->length_recv); - ASSERT_RV(rv); - ASSERT_T0(this->length_recv <= BUFFER_RECV_SIZE); - logRESP(); - sw = (this->buffer_recv[this->length_recv-2]<<8) | this->buffer_recv[this->length_recv-1]; - ASSERT_SW(sw,ok,msk); - return sw; + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE); + ASSERT_X(this->length_recv>=2, "Communication error, less than tow bytes received"); + + this->length_recv -= 2; + this->sw = (this->buffer_recv[length_recv]<<8) | this->buffer_recv[length_recv+1]; + ASSERT_SW(this->sw,ok,msk); + + logRESP(); + return this->sw; } void device_ledger::reset_buffer() { @@ -318,101 +322,25 @@ namespace hw { } const std::string device_ledger::get_name() const { - if (this->full_name.empty() || (this->hCard == 0)) { + if (this->full_name.empty() || !this->connected()) { return std::string("<disconnected:").append(this->name).append(">"); } - return this->full_name; + return this->name; } bool device_ledger::init(void) { #ifdef DEBUG_HWDEVICE this->controle_device = &hw::get_device("default"); #endif - LONG rv; this->release(); - rv = SCardEstablishContext(SCARD_SCOPE_SYSTEM,0,0, &this->hContext); - ASSERT_RV(rv); - MDEBUG( "Device "<<this->id <<" SCardContext created: hContext="<<this->hContext); - this->hCard = 0; - return true; - } - - bool device_ledger::release() { - this->disconnect(); - if (this->hContext) { - SCardReleaseContext(this->hContext); - MDEBUG( "Device "<<this->id <<" SCardContext released: hContext="<<this->hContext); - this->hContext = 0; - this->full_name.clear(); - } + hw_device.init(); + MDEBUG( "Device "<<this->id <<" HIDUSB inited"); return true; } bool device_ledger::connect(void) { - BYTE pbAtr[MAX_ATR_SIZE]; - LPSTR mszReaders; - DWORD dwReaders; - LONG rv; - DWORD dwState, dwProtocol, dwAtrLen, dwReaderLen; - this->disconnect(); -#ifdef SCARD_AUTOALLOCATE - dwReaders = SCARD_AUTOALLOCATE; - rv = SCardListReaders(this->hContext, NULL, (LPSTR)&mszReaders, &dwReaders); -#else - dwReaders = 0; - rv = SCardListReaders(this->hContext, NULL, NULL, &dwReaders); - if (rv != SCARD_S_SUCCESS) - return false; - mszReaders = (LPSTR)calloc(dwReaders, sizeof(char)); - rv = SCardListReaders(this->hContext, NULL, mszReaders, &dwReaders); -#endif - if (rv == SCARD_S_SUCCESS) { - char* p; - const char* prefix = this->name.c_str(); - - p = mszReaders; - MDEBUG( "Looking for " << std::string(prefix)); - while (*p) { - MDEBUG( "Device Found: " << std::string(p)); - if ((strncmp(prefix, p, strlen(prefix))==0)) { - MDEBUG( "Device Match: " << std::string(p)); - if ((rv = SCardConnect(this->hContext, - p, SCARD_SHARE_EXCLUSIVE, SCARD_PROTOCOL_T0, - &this->hCard, &dwProtocol))!=SCARD_S_SUCCESS) { - break; - } - MDEBUG( "Device "<<this->id <<" Connected: hCard="<<this->hCard); - dwAtrLen = sizeof(pbAtr); - if ((rv = SCardStatus(this->hCard, NULL, &dwReaderLen, &dwState, &dwProtocol, pbAtr, &dwAtrLen))!=SCARD_S_SUCCESS) { - break; - } - MDEBUG( "Device "<<this->id <<" Status OK"); - rv = SCARD_S_SUCCESS ; - this->full_name = std::string(p); - break; - } - p += strlen(p) +1; - } - } - - if (mszReaders) { - #ifdef SCARD_AUTOALLOCATE - SCardFreeMemory(this->hContext, mszReaders); - #else - free(mszReaders); - #endif - mszReaders = NULL; - } - if (rv != SCARD_S_SUCCESS) { - if ( hCard) { - SCardDisconnect(this->hCard, SCARD_UNPOWER_CARD); - MDEBUG( "Device "<<this->id <<" disconnected: hCard="<<this->hCard); - this->hCard = 0; - } - } - ASSERT_RV(rv); - + hw_device.connect(0x2c97, 0x0001, 0, 0xffa0); this->reset(); #ifdef DEBUG_HWDEVICE cryptonote::account_public_address pubkey; @@ -422,15 +350,21 @@ namespace hw { crypto::secret_key skey; this->get_secret_keys(vkey,skey); - return rv==SCARD_S_SUCCESS; + return true; + } + + bool device_ledger::connected(void) const { + return hw_device.connected(); } bool device_ledger::disconnect() { - if (this->hCard) { - SCardDisconnect(this->hCard, SCARD_UNPOWER_CARD); - MDEBUG( "Device "<<this->id <<" disconnected: hCard="<<this->hCard); - this->hCard = 0; - } + hw_device.disconnect(); + return true; + } + + bool device_ledger::release() { + this->disconnect(); + hw_device.release(); return true; } @@ -439,20 +373,10 @@ namespace hw { int offset; - reset_buffer(); - switch(mode) { case TRANSACTION_CREATE_REAL: case TRANSACTION_CREATE_FAKE: - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SET_SIGNATURE_MODE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_SET_SIGNATURE_MODE, 1); //account this->buffer_send[offset] = mode; offset += 1; @@ -472,7 +396,7 @@ namespace hw { CHECK_AND_ASSERT_THROW_MES(false, " device_ledger::set_mode(unsigned int mode): invalid mode: "<<mode); } MDEBUG("Switch to mode: " <<mode); - return true; + return device::set_mode(mode); } @@ -483,22 +407,7 @@ namespace hw { bool device_ledger::get_public_address(cryptonote::account_public_address &pubkey){ AUTO_LOCK_CMD(); - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_KEY; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_KEY, 1); memmove(pubkey.m_view_public_key.data, this->buffer_recv, 32); memmove(pubkey.m_spend_public_key.data, this->buffer_recv+32, 32); @@ -514,22 +423,7 @@ namespace hw { memset(skey.data, 0xFF, 32); //spcialkey, normal conf handled in decrypt - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_KEY; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_KEY, 0x02); //View key is retrievied, if allowed, to speed up blockchain parsing memmove(this->viewkey.data, this->buffer_recv+0, 32); @@ -549,34 +443,20 @@ namespace hw { return true; } - bool device_ledger::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) { + bool device_ledger::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE crypto::chacha_key key_x; cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); - this->controle_device->generate_chacha_key(keys_x, key_x); + this->controle_device->generate_chacha_key(keys_x, key_x, kdf_rounds); #endif - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_CHACHA8_PREKEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_CHACHA8_PREKEY); char prekey[200]; memmove(prekey, &this->buffer_recv[0], 200); - crypto::generate_chacha_key_prehashed(&prekey[0], sizeof(prekey), key); + crypto::generate_chacha_key_prehashed(&prekey[0], sizeof(prekey), key, kdf_rounds); #ifdef DEBUG_HWDEVICE hw::ledger::check32("generate_chacha_key_prehashed", "key", (char*)key_x.data(), (char*)key.data()); @@ -615,19 +495,7 @@ namespace hw { crypto::derive_subaddress_public_key(pub, derivation, output_index,derived_pub); } else { - int offset =0; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_SUBADDRESS_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_SUBADDRESS_PUBLIC_KEY); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -658,7 +526,6 @@ namespace hw { crypto::public_key device_ledger::get_subaddress_spend_public_key(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); crypto::public_key D; - int offset; #ifdef DEBUG_HWDEVICE const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); @@ -675,17 +542,7 @@ namespace hw { D = keys.m_account_address.m_spend_public_key; } else { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS_SPEND_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS_SPEND_PUBLIC_KEY); //index static_assert(sizeof(cryptonote::subaddress_index) == 8, "cryptonote::subaddress_index shall be 8 bytes length"); memmove(this->buffer_send+offset, &index, sizeof(cryptonote::subaddress_index)); @@ -720,7 +577,6 @@ namespace hw { cryptonote::account_public_address device_ledger::get_subaddress(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); cryptonote::account_public_address address; - int offset; #ifdef DEBUG_HWDEVICE const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); @@ -739,17 +595,7 @@ namespace hw { if (index.is_zero()) { address = keys.m_account_address; } else { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS); //index static_assert(sizeof(cryptonote::subaddress_index) == 8, "cryptonote::subaddress_index shall be 8 bytes length"); memmove(this->buffer_send+offset, &index, sizeof(cryptonote::subaddress_index)); @@ -774,7 +620,6 @@ namespace hw { crypto::secret_key device_ledger::get_subaddress_secret_key(const crypto::secret_key &sec, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); crypto::secret_key sub_sec; - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key sec_x = hw::ledger::decrypt(sec); @@ -786,17 +631,7 @@ namespace hw { hw::ledger::log_hexbuffer("get_subaddress_secret_key: [[OUT]] sub_sec", sub_sec_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS_SECRET_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS_SECRET_KEY); //sec memmove(this->buffer_send+offset, sec.data, 32); offset += 32; @@ -827,16 +662,7 @@ namespace hw { AUTO_LOCK_CMD(); int offset, sw; - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VERIFY_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_VERIFY_KEY); //sec memmove(this->buffer_send+offset, secret_key.data, 32); offset += 32; @@ -859,7 +685,6 @@ namespace hw { bool device_ledger::scalarmultKey(rct::key & aP, const rct::key &P, const rct::key &a) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key P_x = P; @@ -871,17 +696,7 @@ namespace hw { hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aP", (char*)aP_x.bytes, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_SCAL_MUL_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_KEY); //pub memmove(this->buffer_send+offset, P.bytes, 32); offset += 32; @@ -906,7 +721,6 @@ namespace hw { bool device_ledger::scalarmultBase(rct::key &aG, const rct::key &a) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key a_x = hw::ledger::decrypt(a); @@ -916,17 +730,7 @@ namespace hw { hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aG", (char*)aG_x.bytes, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_SCAL_MUL_BASE; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_BASE); //sec memmove(this->buffer_send+offset, a.bytes, 32); offset += 32; @@ -947,7 +751,6 @@ namespace hw { bool device_ledger::sc_secret_add( crypto::secret_key &r, const crypto::secret_key &a, const crypto::secret_key &b) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key a_x = hw::ledger::decrypt(a); @@ -956,17 +759,7 @@ namespace hw { this->controle_device->sc_secret_add(r_x, a_x, b_x); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_KEY_ADD; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_KEY_ADD); //sec key memmove(this->buffer_send+offset, a.data, 32); offset += 32; @@ -995,8 +788,6 @@ namespace hw { throw std::runtime_error("device generate key does not support recover"); } - int offset; - #ifdef DEBUG_HWDEVICE bool recover_x = recover; const crypto::secret_key recovery_key_x = recovery_key; @@ -1004,21 +795,7 @@ namespace hw { crypto::secret_key sec_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GENERATE_KEYPAIR; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GENERATE_KEYPAIR); //pub key memmove(pub.data, &this->buffer_recv[0], 32); @@ -1058,19 +835,7 @@ namespace hw { r = crypto::generate_key_derivation(pub, this->viewkey, derivation); } else { - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GEN_KEY_DERIVATION; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GEN_KEY_DERIVATION); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -1119,7 +884,6 @@ namespace hw { bool device_ledger::derivation_to_scalar(const crypto::key_derivation &derivation, const size_t output_index, crypto::ec_scalar &res) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); @@ -1131,17 +895,7 @@ namespace hw { hw::ledger::log_hexbuffer("derivation_to_scalar: [[OUT]] res ", res_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVATION_TO_SCALAR; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVATION_TO_SCALAR); //derivattion memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1169,7 +923,6 @@ namespace hw { bool device_ledger::derive_secret_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::secret_key &sec, crypto::secret_key &derived_sec) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); @@ -1183,17 +936,7 @@ namespace hw { hw::ledger::log_hexbuffer("derive_secret_key: [[OUT]] derived_sec", derived_sec_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_SECRET_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_SECRET_KEY); //derivation memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1224,7 +967,6 @@ namespace hw { bool device_ledger::derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub, crypto::public_key &derived_pub){ AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); @@ -1238,17 +980,7 @@ namespace hw { hw::ledger::log_hexbuffer("derive_public_key: [[OUT]] derived_pub ", derived_pub_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_PUBLIC_KEY); //derivation memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1278,7 +1010,6 @@ namespace hw { bool device_ledger::secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key sec_x = hw::ledger::decrypt(sec); @@ -1291,17 +1022,7 @@ namespace hw { } #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_KEY_TO_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_KEY_TO_PUBLIC_KEY); //sec key memmove(this->buffer_send+offset, sec.data, 32); offset += 32; @@ -1322,7 +1043,6 @@ namespace hw { bool device_ledger::generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image){ AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::public_key pub_x = pub; @@ -1334,17 +1054,7 @@ namespace hw { hw::ledger::log_hexbuffer("generate_key_image: [[OUT]] image ", image_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GEN_KEY_IMAGE; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GEN_KEY_IMAGE); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -1372,20 +1082,10 @@ namespace hw { bool device_ledger::open_tx(crypto::secret_key &tx_key) { AUTO_LOCK_CMD(); - int offset; - reset_buffer(); key_map.clear(); + int offset = set_command_header_noopt(INS_OPEN_TX, 0x01); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_OPEN_TX; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; //account this->buffer_send[offset+0] = 0x00; this->buffer_send[offset+1] = 0x00; @@ -1404,7 +1104,6 @@ namespace hw { bool device_ledger::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::public_key public_key_x = public_key; @@ -1413,17 +1112,7 @@ namespace hw { this->controle_device->encrypt_payment_id(payment_id_x, public_key_x, secret_key_x); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_STEALTH; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_STEALTH); //pub memmove(&this->buffer_send[offset], public_key.data, 32); offset += 32; @@ -1455,7 +1144,6 @@ namespace hw { bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); @@ -1463,17 +1151,7 @@ namespace hw { this->controle_device->ecdhEncode(unmasked_x, AKout_x); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_BLIND; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_BLIND); // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1503,7 +1181,6 @@ namespace hw { bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); @@ -1511,17 +1188,8 @@ namespace hw { this->controle_device->ecdhDecode(masked_x, AKout_x); #endif - reset_buffer(); + int offset = set_command_header_noopt(INS_UNBLIND); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_UNBLIND; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1571,16 +1239,7 @@ namespace hw { data = blob.data(); // ====== u8 type, varint txnfee ====== - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x01; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_VALIDATE, 0x01, 0x01); //options this->buffer_send[offset] = (inputs_size == 0)?0x00:0x80; offset += 1; @@ -1606,15 +1265,9 @@ namespace hw { this->exchange(); //pseudoOuts - if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeSimpleBulletproof)) { + if (type == rct::RCTTypeSimple) { for ( i = 0; i < inputs_size; i++) { - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = i+2; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options this->buffer_send[offset] = (i==inputs_size-1)? 0x00:0x80; offset += 1; @@ -1641,14 +1294,7 @@ namespace hw { log_hexbuffer("Pout not found", (char*)outPk[i].dest.bytes, 32); CHECK_AND_ASSERT_THROW_MES(found, "Pout not found"); } - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x02, i+1); //options this->buffer_send[offset] = (i==outputs_size-1)? 0x00:0x80 ; offset += 1; @@ -1693,14 +1339,7 @@ namespace hw { // ====== C[], message, proof====== C_offset = kv_offset; for (i = 0; i < outputs_size; i++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x03, i+1); //options this->buffer_send[offset] = 0x80 ; offset += 1; @@ -1715,17 +1354,7 @@ namespace hw { } - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_VALIDATE, 0x03, i+1); //message memmove(this->buffer_send+offset, hashes[0].bytes,32); offset += 32; @@ -1750,7 +1379,6 @@ namespace hw { bool device_ledger::mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &II) { AUTO_LOCK_CMD(); - int offset; unsigned char options; #ifdef DEBUG_HWDEVICE @@ -1762,17 +1390,7 @@ namespace hw { rct::key II_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_MLSAG, 0x01); //value H memmove(this->buffer_send+offset, H.bytes, 32); offset += 32; @@ -1805,7 +1423,6 @@ namespace hw { bool device_ledger::mlsag_prepare(rct::key &a, rct::key &aG) { AUTO_LOCK_CMD(); - int offset; unsigned char options; #ifdef DEBUG_HWDEVICE @@ -1813,21 +1430,7 @@ namespace hw { rct::key aG_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_MLSAG, 0x01); memmove(a.bytes, &this->buffer_recv[32*0], 32); memmove(aG.bytes, &this->buffer_recv[32*1], 32); @@ -1843,7 +1446,6 @@ namespace hw { bool device_ledger::mlsag_hash(const rct::keyV &long_message, rct::key &c) { AUTO_LOCK_CMD(); - int offset; unsigned char options; size_t cnt; @@ -1855,14 +1457,7 @@ namespace hw { cnt = long_message.size(); for (size_t i = 0; i<cnt; i++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_MLSAG, 0x02, i+1); //options this->buffer_send[offset] = (i==(cnt-1))?0x00:0x80; //last @@ -1887,7 +1482,6 @@ namespace hw { bool device_ledger::mlsag_sign(const rct::key &c, const rct::keyV &xx, const rct::keyV &alpha, const size_t rows, const size_t dsRows, rct::keyV &ss) { AUTO_LOCK_CMD(); - int offset; CHECK_AND_ASSERT_THROW_MES(dsRows<=rows, "dsRows greater than rows"); CHECK_AND_ASSERT_THROW_MES(xx.size() == rows, "xx size does not match rows"); @@ -1905,14 +1499,7 @@ namespace hw { #endif for (size_t j = 0; j < dsRows; j++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = j+1; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_MLSAG, 0x03, j+1); //options this->buffer_send[offset] = 0x00; if (j==(dsRows-1)) { @@ -1949,24 +1536,7 @@ namespace hw { bool device_ledger::close_tx() { AUTO_LOCK_CMD(); - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_CLOSE_TX; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); - + send_simple(INS_CLOSE_TX); return true; } diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index f1fcaab87..2f5beb044 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -33,8 +33,7 @@ #include <cstddef> #include <string> #include "device.hpp" -#include <PCSC/winscard.h> -#include <PCSC/wintypes.h> +#include "device_io_hid.hpp" #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> @@ -84,19 +83,23 @@ namespace hw { mutable boost::recursive_mutex device_locker; mutable boost::mutex command_locker; - //PCSC management - std::string full_name; - SCARDCONTEXT hContext; - SCARDHANDLE hCard; - DWORD length_send; - BYTE buffer_send[BUFFER_SEND_SIZE]; - DWORD length_recv; - BYTE buffer_recv[BUFFER_RECV_SIZE]; - unsigned int id; + //IO + hw::io::device_io_hid hw_device; + std::string full_name; + unsigned int length_send; + unsigned char buffer_send[BUFFER_SEND_SIZE]; + unsigned int length_recv; + unsigned char buffer_recv[BUFFER_RECV_SIZE]; + unsigned int sw; + unsigned int id; void logCMD(void); void logRESP(void); - unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); + unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); void reset_buffer(void); + int set_command_header(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); + int set_command_header_noopt(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); + void send_simple(unsigned char ins, unsigned char p1 = 0x00); + // hw running mode device_mode mode; @@ -119,7 +122,7 @@ namespace hw { device_ledger(const device_ledger &device) = delete ; device_ledger& operator=(const device_ledger &device) = delete; - explicit operator bool() const override {return this->hContext != 0;} + explicit operator bool() const override {return this->connected(); } bool reset(void); @@ -133,8 +136,12 @@ namespace hw { bool release() override; bool connect(void) override; bool disconnect() override; + bool connected(void) const; + + bool set_mode(device_mode mode) override; - bool set_mode(device_mode mode) override; + device_type get_type() const override {return device_type::LEDGER;}; + device_protocol_t device_protocol() const override { return PROTOCOL_PROXY; }; /* ======================================================================= */ /* LOCKER */ @@ -148,7 +155,7 @@ namespace hw { /* ======================================================================= */ bool get_public_address(cryptonote::account_public_address &pubkey) override; bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; - bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) override; + bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) override; /* ======================================================================= */ diff --git a/src/device/log.cpp b/src/device/log.cpp index cbbcfc953..c9d3b551b 100644 --- a/src/device/log.cpp +++ b/src/device/log.cpp @@ -32,29 +32,34 @@ namespace hw { - #ifdef WITH_DEVICE_LEDGER - namespace ledger { - - #undef MONERO_DEFAULT_LOG_CATEGORY - #define MONERO_DEFAULT_LOG_CATEGORY "device.ledger" + #undef MONERO_DEFAULT_LOG_CATEGORY + #define MONERO_DEFAULT_LOG_CATEGORY "device" - void buffer_to_str(char *to_buff, size_t to_len, const char *buff, size_t len) { - CHECK_AND_ASSERT_THROW_MES(to_len > (len*2), "destination buffer too short. At least" << (len*2+1) << " bytes required"); - for (size_t i=0; i<len; i++) { - sprintf(to_buff+2*i, "%.02x", (unsigned char)buff[i]); - } + void buffer_to_str(char *to_buff, size_t to_len, const char *buff, size_t len) { + CHECK_AND_ASSERT_THROW_MES(to_len > (len*2), "destination buffer too short. At least" << (len*2+1) << " bytes required"); + for (size_t i=0; i<len; i++) { + sprintf(to_buff+2*i, "%.02x", (unsigned char)buff[i]); } + } - void log_hexbuffer(std::string msg, const char* buff, size_t len) { - char logstr[1025]; - buffer_to_str(logstr, sizeof(logstr), buff, len); - MDEBUG(msg<< ": " << logstr); - } + void log_hexbuffer(const std::string &msg, const char* buff, size_t len) { + char logstr[1025]; + buffer_to_str(logstr, sizeof(logstr), buff, len); + MDEBUG(msg<< ": " << logstr); + } - void log_message(std::string msg, std::string info ) { - MDEBUG(msg << ": " << info); - } + void log_message(const std::string &msg, const std::string &info ) { + MDEBUG(msg << ": " << info); + } + + + #ifdef WITH_DEVICE_LEDGER + namespace ledger { + + #undef MONERO_DEFAULT_LOG_CATEGORY + #define MONERO_DEFAULT_LOG_CATEGORY "device.ledger" + #ifdef DEBUG_HWDEVICE extern crypto::secret_key dbg_viewkey; extern crypto::secret_key dbg_spendkey; @@ -122,16 +127,18 @@ namespace hw { rct::keyV decrypt(const rct::keyV &keys) { rct::keyV x ; + x.reserve(keys.size()); for (unsigned int j = 0; j<keys.size(); j++) { x.push_back(decrypt(keys[j])); } return x; } - static void check(std::string msg, std::string info, const char *h, const char *d, int len, bool crypted) { + static void check(const std::string &msg, const std::string &info, const char *h, const char *d, size_t len, bool crypted) { char dd[32]; char logstr[128]; + CHECK_AND_ASSERT_THROW_MES(len <= sizeof(dd), "invalid len"); memmove(dd,d,len); if (crypted) { CHECK_AND_ASSERT_THROW_MES(len<=32, "encrypted data greater than 32"); @@ -149,11 +156,11 @@ namespace hw { } } - void check32(std::string msg, std::string info, const char *h, const char *d, bool crypted) { + void check32(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) { check(msg, info, h, d, 32, crypted); } - void check8(std::string msg, std::string info, const char *h, const char *d, bool crypted) { + void check8(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) { check(msg, info, h, d, 8, crypted); } #endif diff --git a/src/device/log.hpp b/src/device/log.hpp index 1ab572c2c..25a214a6c 100644 --- a/src/device/log.hpp +++ b/src/device/log.hpp @@ -40,12 +40,13 @@ namespace hw { + void buffer_to_str(char *to_buff, size_t to_len, const char *buff, size_t len) ; + void log_hexbuffer(const std::string &msg, const char* buff, size_t len); + void log_message(const std::string &msg, const std::string &info ); + #ifdef WITH_DEVICE_LEDGER namespace ledger { - void buffer_to_str(char *to_buff, size_t to_len, const char *buff, size_t len) ; - void log_hexbuffer(std::string msg, const char* buff, size_t len); - void log_message(std::string msg, std::string info ); #ifdef DEBUG_HWDEVICE #define TRACK printf("file %s:%d\n",__FILE__, __LINE__) //#define TRACK MCDEBUG("ledger"," At file " << __FILE__ << ":" << __LINE__) @@ -59,8 +60,8 @@ namespace hw { crypto::ec_scalar decrypt(const crypto::ec_scalar &res); rct::keyV decrypt(const rct::keyV &keys); - void check32(std::string msg, std::string info, const char *h, const char *d, bool crypted=false); - void check8(std::string msg, std::string info, const char *h, const char *d, bool crypted=false); + void check32(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false); + void check8(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false); void set_check_verbose(bool verbose); #endif diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt new file mode 100644 index 000000000..c555e9fcd --- /dev/null +++ b/src/device_trezor/CMakeLists.txt @@ -0,0 +1,123 @@ +# Copyright (c) 2014-2017, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(TREZOR_PROTOB_H + trezor/messages/messages.pb.h + trezor/messages/messages-common.pb.h + trezor/messages/messages-management.pb.h + trezor/messages/messages-monero.pb.h +) + +set(TREZOR_PROTOB_CPP + trezor/messages/messages.pb.cc + trezor/messages/messages-common.pb.cc + trezor/messages/messages-management.pb.cc + trezor/messages/messages-monero.pb.cc +) + +set(trezor_headers + trezor/exceptions.hpp + trezor/messages_map.hpp + trezor/protocol.hpp + trezor/transport.hpp + device_trezor_base.hpp + device_trezor.hpp + trezor.hpp + ${TREZOR_PROTOB_H} +) + +set(trezor_sources + trezor/messages_map.cpp + trezor/protocol.cpp + trezor/transport.cpp + device_trezor_base.cpp + device_trezor.cpp + ${TREZOR_PROTOB_CPP} +) + +set(trezor_private_headers) + + +include(FindProtobuf) +find_package(Protobuf) # REQUIRED + +# Test for HAVE_PROTOBUF from the parent +if(Protobuf_FOUND AND HAVE_PROTOBUF) + if ("$ENV{PYTHON3}" STREQUAL "") + set(PYTHON3 "python3") + else() + set(PYTHON3 "$ENV{PYTHON3}" CACHE INTERNAL "Copied from environment variable") + endif() + + execute_process(COMMAND ${PYTHON3} tools/build_protob.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) + if(RET) + message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})." + "OUT: ${OUT}, ERR: ${ERR}." + "Please read src/device_trezor/trezor/tools/README.md") + else() + message(STATUS "Trezor protobuf messages regenerated ${OUT}") + set(TREZOR_PROTOBUF_GENERATED 1) + endif() +endif() + + +if(TREZOR_PROTOBUF_GENERATED) + message(STATUS "Trezor support enabled") + + add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) + + monero_private_headers(device_trezor + ${device_private_headers} + ${PROTOBUF_INCLUDE_DIR}) + + monero_add_library(device_trezor + ${trezor_sources} + ${trezor_headers} + ${trezor_private_headers}) + + target_link_libraries(device_trezor + PUBLIC + device + cncrypto + ringct_basic + cryptonote_core + common + ${SODIUM_LIBRARY} + ${Boost_CHRONO_LIBRARY} + ${PROTOBUF_LIBRARY} + PRIVATE + ${EXTRA_LIBRARIES}) + + # set(WITH_DEVICE_TREZOR 1 PARENT_SCOPE) + # add_definitions(-DWITH_DEVICE_TREZOR=1) + +else() + monero_private_headers(device_trezor) + monero_add_library(device_trezor device_trezor.cpp) + target_link_libraries(device_trezor PUBLIC cncrypto) +endif() diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp new file mode 100644 index 000000000..07c03fc66 --- /dev/null +++ b/src/device_trezor/device_trezor.cpp @@ -0,0 +1,363 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "device_trezor.hpp" + +namespace hw { +namespace trezor { + +#if WITH_DEVICE_TREZOR + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" + +#define HW_TREZOR_NAME "Trezor" +#define HW_TREZOR_NAME_LITE "TrezorLite" + + static device_trezor *trezor_device = nullptr; + static device_trezor *ensure_trezor_device(){ + if (!trezor_device) { + trezor_device = new device_trezor(); + trezor_device->set_name(HW_TREZOR_NAME); + } + return trezor_device; + } + + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry) { + registry.insert(std::make_pair(HW_TREZOR_NAME, std::unique_ptr<device>(ensure_trezor_device()))); + } + + void register_all() { + hw::register_device(HW_TREZOR_NAME, ensure_trezor_device()); + } + + device_trezor::device_trezor() { + + } + + device_trezor::~device_trezor() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MWARNING("Could not disconnect and release: " << e.what()); + } + } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + + bool device_trezor::get_public_address(cryptonote::account_public_address &pubkey) { + try { + auto res = get_address(); + + cryptonote::address_parse_info info{}; + bool r = cryptonote::get_account_address_from_str(info, this->network_type, res->address()); + CHECK_AND_ASSERT_MES(r, false, "Could not parse returned address. Address parse failed: " + res->address()); + CHECK_AND_ASSERT_MES(!info.is_subaddress, false, "Trezor returned a sub address"); + + pubkey = info.address; + return true; + + } catch(std::exception const& e){ + MERROR("Get public address exception: " << e.what()); + return false; + } + } + + bool device_trezor::get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) { + try { + MDEBUG("Loading view-only key from the Trezor. Please check the Trezor for a confirmation."); + auto res = get_view_key(); + CHECK_AND_ASSERT_MES(res->watch_key().size() == 32, false, "Trezor returned invalid view key"); + + spendkey = crypto::null_skey; // not given + memcpy(viewkey.data, res->watch_key().data(), 32); + + return true; + + } catch(std::exception const& e){ + MERROR("Get secret keys exception: " << e.what()); + return false; + } + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + std::shared_ptr<messages::monero::MoneroAddress> device_trezor::get_address( + const boost::optional<std::vector<uint32_t>> & path, + const boost::optional<cryptonote::network_type> & network_type){ + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + auto req = std::make_shared<messages::monero::MoneroGetAddress>(); + this->set_msg_addr<messages::monero::MoneroGetAddress>(req.get(), path, network_type); + + auto response = this->client_exchange<messages::monero::MoneroAddress>(req); + MTRACE("Get address response received"); + return response; + } + + std::shared_ptr<messages::monero::MoneroWatchKey> device_trezor::get_view_key( + const boost::optional<std::vector<uint32_t>> & path, + const boost::optional<cryptonote::network_type> & network_type){ + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + auto req = std::make_shared<messages::monero::MoneroGetWatchKey>(); + this->set_msg_addr<messages::monero::MoneroGetWatchKey>(req.get(), path, network_type); + + auto response = this->client_exchange<messages::monero::MoneroWatchKey>(req); + MTRACE("Get watch key response received"); + return response; + } + + void device_trezor::ki_sync(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + hw::device_cold::exported_key_image & ski) + { + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> req; + + std::vector<protocol::ki::MoneroTransferDetails> mtds; + std::vector<protocol::ki::MoneroExportedKeyImage> kis; + protocol::ki::key_image_data(wallet, transfers, mtds); + protocol::ki::generate_commitment(mtds, transfers, req); + + this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get()); + auto ack1 = this->client_exchange<messages::monero::MoneroKeyImageExportInitAck>(req); + + const auto batch_size = 10; + const auto num_batches = (mtds.size() + batch_size - 1) / batch_size; + for(uint64_t cur = 0; cur < num_batches; ++cur){ + auto step_req = std::make_shared<messages::monero::MoneroKeyImageSyncStepRequest>(); + auto idx_finish = std::min(static_cast<uint64_t>((cur + 1) * batch_size), static_cast<uint64_t>(mtds.size())); + for(uint64_t idx = cur * batch_size; idx < idx_finish; ++idx){ + auto added_tdis = step_req->add_tdis(); + CHECK_AND_ASSERT_THROW_MES(idx < mtds.size(), "Invalid transfer detail index"); + *added_tdis = mtds[idx]; + } + + auto step_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncStepAck>(step_req); + auto kis_size = step_ack->kis_size(); + kis.reserve(static_cast<size_t>(kis_size)); + for(int i = 0; i < kis_size; ++i){ + auto ckis = step_ack->kis(i); + kis.push_back(ckis); + } + + MTRACE("Batch " << cur << " / " << num_batches << " batches processed"); + } + + auto final_req = std::make_shared<messages::monero::MoneroKeyImageSyncFinalRequest>(); + auto final_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncFinalAck>(final_req); + ski.reserve(kis.size()); + + for(auto & sub : kis){ + char buff[32*3]; + protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(), + reinterpret_cast<const uint8_t *>(final_ack->enc_key().data()), + reinterpret_cast<const uint8_t *>(sub.iv().data()), buff); + + ::crypto::signature sig{}; + ::crypto::key_image ki; + memcpy(ki.data, buff, 32); + memcpy(sig.c.data, buff + 32, 32); + memcpy(sig.r.data, buff + 64, 32); + ski.push_back(std::make_pair(ki, sig)); + } + } + + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) + { + size_t num_tx = unsigned_tx.txes.size(); + signed_tx.key_images.clear(); + signed_tx.key_images.resize(unsigned_tx.transfers.size()); + + for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { + std::shared_ptr<protocol::tx::Signer> signer; + tx_sign(wallet, unsigned_tx, tx_idx, aux_data, signer); + + auto & cdata = signer->tdata(); + auto aux_info_cur = signer->store_tx_aux_info(); + aux_data.tx_device_aux.emplace_back(aux_info_cur); + + // Pending tx reconstruction + signed_tx.ptx.emplace_back(); + auto & cpend = signed_tx.ptx.back(); + cpend.tx = cdata.tx; + cpend.dust = 0; + cpend.fee = 0; + cpend.dust_added_to_fee = false; + cpend.change_dts = cdata.tx_data.change_dts; + cpend.selected_transfers = cdata.tx_data.selected_transfers; + cpend.key_images = ""; + cpend.dests = cdata.tx_data.dests; + cpend.construction_data = cdata.tx_data; + + // Transaction check + cryptonote::blobdata tx_blob; + cryptonote::transaction tx_deserialized; + bool r = cryptonote::t_serializable_object_to_blob(cpend.tx, tx_blob); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed"); + r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed"); + + std::string key_images; + bool all_are_txin_to_key = std::all_of(cdata.tx.vin.begin(), cdata.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool + { + CHECKED_GET_SPECIFIC_VARIANT(s_e, const cryptonote::txin_to_key, in, false); + key_images += boost::to_string(in.k_image) + " "; + return true; + }); + if(!all_are_txin_to_key) { + throw std::invalid_argument("Not all are txin_to_key"); + } + cpend.key_images = key_images; + + // KI sync + size_t num_sources = cdata.tx_data.sources.size(); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size"); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size"); + for(size_t src_idx = 0; src_idx < num_sources; ++src_idx){ + size_t idx_mapped = cdata.source_permutation[src_idx]; + CHECK_AND_ASSERT_THROW_MES(idx_mapped < cdata.tx_data.selected_transfers.size(), "Invalid idx_mapped"); + CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); + + size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; + auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); + + CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); + signed_tx.key_images[idx_map_src] = vini.k_image; + } + } + } + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer) + { + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); + signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data); + const tools::wallet2::tx_construction_data & cur_tx = unsigned_tx.txes[idx]; + unsigned long num_sources = cur_tx.sources.size(); + unsigned long num_outputs = cur_tx.splitted_dsts.size(); + + // Step: Init + auto init_msg = signer->step_init(); + this->set_msg_addr(init_msg.get()); + + auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg); + signer->step_init_ack(response); + + // Step: Set transaction inputs + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionSetInputAck>(src); + signer->step_set_input_ack(ack); + } + + // Step: sort + auto perm_req = signer->step_permutation(); + if (perm_req){ + auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req); + signer->step_permutation_ack(perm_ack); + } + + // Step: input_vini + if (!signer->in_memory()){ + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_vini_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); + signer->step_set_vini_input_ack(ack); + } + } + + // Step: all inputs set + auto all_inputs_set = signer->step_all_inputs_set(); + auto ack_all_inputs = this->client_exchange<messages::monero::MoneroTransactionAllInputsSetAck>(all_inputs_set); + signer->step_all_inputs_set_ack(ack_all_inputs); + + // Step: outputs + for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){ + auto src = signer->step_set_output(cur_dst); + auto ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(src); + signer->step_set_output_ack(ack); + } + + // Step: all outs set + auto all_out_set = signer->step_all_outs_set(); + auto ack_all_out_set = this->client_exchange<messages::monero::MoneroTransactionAllOutSetAck>(all_out_set); + signer->step_all_outs_set_ack(ack_all_out_set, *this); + + // Step: sign each input + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_sign_input(cur_src); + auto ack_sign = this->client_exchange<messages::monero::MoneroTransactionSignInputAck>(src); + signer->step_sign_input_ack(ack_sign); + } + + // Step: final + auto final_msg = signer->step_final(); + auto ack_final = this->client_exchange<messages::monero::MoneroTransactionFinalAck>(final_msg); + signer->step_final_ack(ack_final); + } + +#else //WITH_DEVICE_TREZOR + + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry) { + } + + void register_all() { + } + +#endif //WITH_DEVICE_TREZOR +}}
\ No newline at end of file diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp new file mode 100644 index 000000000..765c9b82c --- /dev/null +++ b/src/device_trezor/device_trezor.hpp @@ -0,0 +1,132 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_H +#define MONERO_DEVICE_TREZOR_H + + +#include <cstddef> +#include <string> +#include "device/device.hpp" +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include <boost/scope_exit.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> +#include "cryptonote_config.h" +#include "trezor.hpp" +#include "device_trezor_base.hpp" + +namespace hw { +namespace trezor { + + void register_all(); + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry); + +#if WITH_DEVICE_TREZOR + class device_trezor; + + /** + * Main device + */ + class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { + protected: + // To speed up blockchain parsing the view key maybe handle here. + crypto::secret_key viewkey; + bool has_view_key; + + public: + device_trezor(); + virtual ~device_trezor() override; + + device_trezor(const device_trezor &device) = delete ; + device_trezor& operator=(const device_trezor &device) = delete; + + explicit operator bool() const override {return true;} + + device_protocol_t device_protocol() const override { return PROTOCOL_COLD; }; + + bool has_ki_cold_sync() const override { return true; } + bool has_tx_cold_sign() const override { return true; } + void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + bool get_public_address(cryptonote::account_public_address &pubkey) override; + bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Get address. Throws. + */ + std::shared_ptr<messages::monero::MoneroAddress> get_address( + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none); + + /** + * Get watch key from device. Throws. + */ + std::shared_ptr<messages::monero::MoneroWatchKey> get_view_key( + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none); + + /** + * Key image sync with the Trezor. + */ + void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + hw::device_cold::exported_key_image & ski) override; + + /** + * Signs particular transaction idx in the unsigned set, keeps state in the signer + */ + void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer); + + /** + * Signs unsigned transaction with the Trezor. + */ + void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) override; + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_H diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp new file mode 100644 index 000000000..3a98bba5a --- /dev/null +++ b/src/device_trezor/device_trezor_base.cpp @@ -0,0 +1,301 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "device_trezor_base.hpp" + +namespace hw { +namespace trezor { + +#if WITH_DEVICE_TREZOR + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_button_request(const messages::common::ButtonRequest * msg){ + MDEBUG("on_button_request"); + device.on_button_request(); + return std::make_shared<messages::common::ButtonAck>(); + } + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_pin_matrix_request(const messages::common::PinMatrixRequest * msg){ + MDEBUG("on_pin_request"); + epee::wipeable_string pin; + device.on_pin_request(pin); + auto resp = std::make_shared<messages::common::PinMatrixAck>(); + resp->set_pin(pin.data(), pin.size()); + return resp; + } + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_passphrase_request(const messages::common::PassphraseRequest * msg){ + MDEBUG("on_passhprase_request"); + epee::wipeable_string passphrase; + device.on_passphrase_request(msg->on_device(), passphrase); + auto resp = std::make_shared<messages::common::PassphraseAck>(); + if (!msg->on_device()){ + resp->set_passphrase(passphrase.data(), passphrase.size()); + } + return resp; + } + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_passphrase_state_request(const messages::common::PassphraseStateRequest * msg){ + MDEBUG("on_passhprase_state_request"); + device.on_passphrase_state_request(msg->state()); + return std::make_shared<messages::common::PassphraseStateAck>(); + } + + const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080, 0x80000000}; + + device_trezor_base::device_trezor_base() { + + } + + device_trezor_base::~device_trezor_base() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MERROR("Could not disconnect and release: " << e.what()); + } + } + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + + bool device_trezor_base::reset() { + return false; + } + + bool device_trezor_base::set_name(const std::string & name) { + this->full_name = name; + this->name = ""; + + auto delim = name.find(':'); + if (delim != std::string::npos && delim + 1 < name.length()) { + this->name = name.substr(delim + 1); + } + + return true; + } + + const std::string device_trezor_base::get_name() const { + if (this->full_name.empty()) { + return std::string("<disconnected:").append(this->name).append(">"); + } + return this->full_name; + } + + bool device_trezor_base::init() { + if (!release()){ + MERROR("Release failed"); + return false; + } + + if (!m_protocol_callback){ + m_protocol_callback = std::make_shared<trezor_protocol_callback>(*this); + } + return true; + } + + bool device_trezor_base::release() { + try { + disconnect(); + return true; + + } catch(std::exception const& e){ + MERROR("Release exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::connect() { + disconnect(); + + // Enumerate all available devices + try { + hw::trezor::t_transport_vect trans; + + MDEBUG("Enumerating Trezor devices..."); + enumerate(trans); + + MDEBUG("Enumeration yielded " << trans.size() << " devices"); + for (auto &cur : trans) { + MDEBUG(" device: " << *(cur.get())); + std::string cur_path = cur->get_path(); + if (boost::starts_with(cur_path, this->name)) { + MDEBUG("Device Match: " << cur_path); + m_transport = cur; + break; + } + } + + if (!m_transport) { + MERROR("No matching Trezor device found. Device specifier: \"" + this->name + "\""); + return false; + } + + m_transport->open(); + return true; + + } catch(std::exception const& e){ + MERROR("Open exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::disconnect() { + if (m_transport){ + try { + m_transport->close(); + m_transport = nullptr; + + } catch(std::exception const& e){ + MERROR("Disconnect exception: " << e.what()); + m_transport = nullptr; + return false; + } + } + return true; + } + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + + //lock the device for a long sequence + void device_trezor_base::lock() { + MTRACE("Ask for LOCKING for device " << this->name << " in thread "); + device_locker.lock(); + MTRACE("Device " << this->name << " LOCKed"); + } + + //lock the device for a long sequence + bool device_trezor_base::try_lock() { + MTRACE("Ask for LOCKING(try) for device " << this->name << " in thread "); + bool r = device_locker.try_lock(); + if (r) { + MTRACE("Device " << this->name << " LOCKed(try)"); + } else { + MDEBUG("Device " << this->name << " not LOCKed(try)"); + } + return r; + } + + //unlock the device + void device_trezor_base::unlock() { + MTRACE("Ask for UNLOCKING for device " << this->name << " in thread "); + device_locker.unlock(); + MTRACE("Device " << this->name << " UNLOCKed"); + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + void device_trezor_base::require_connected(){ + if (!m_transport){ + throw exc::NotConnectedException(); + } + } + + void device_trezor_base::call_ping_unsafe(){ + auto pingMsg = std::make_shared<messages::management::Ping>(); + pingMsg->set_message("PING"); + + auto success = this->client_exchange<messages::common::Success>(pingMsg); // messages::MessageType_Success + MDEBUG("Ping response " << success->message()); + (void)success; + } + + void device_trezor_base::test_ping(){ + require_connected(); + + try { + this->call_ping_unsafe(); + + } catch(exc::TrezorException const& e){ + MINFO("Trezor does not respond: " << e.what()); + throw exc::DeviceNotResponsiveException(std::string("Trezor not responding: ") + e.what()); + } + } + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + bool device_trezor_base::ping() { + AUTO_LOCK_CMD(); + if (!m_transport){ + MINFO("Ping failed, device not connected"); + return false; + } + + try { + this->call_ping_unsafe(); + return true; + + } catch(std::exception const& e) { + MERROR("Ping failed, exception thrown " << e.what()); + } catch(...){ + MERROR("Ping failed, general exception thrown" << boost::current_exception_diagnostic_information()); + } + + return false; + } + + void device_trezor_base::on_button_request() + { + if (m_callback){ + m_callback->on_button_request(); + } + } + + void device_trezor_base::on_pin_request(epee::wipeable_string & pin) + { + if (m_callback){ + m_callback->on_pin_request(pin); + } + } + + void device_trezor_base::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) + { + if (m_callback){ + m_callback->on_passphrase_request(on_device, passphrase); + } + } + + void device_trezor_base::on_passphrase_state_request(const std::string & state) + { + if (m_callback){ + m_callback->on_passphrase_state_request(state); + } + } + +#endif //WITH_DEVICE_TREZOR +}} diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp new file mode 100644 index 000000000..644a49332 --- /dev/null +++ b/src/device_trezor/device_trezor_base.hpp @@ -0,0 +1,301 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_BASE_H +#define MONERO_DEVICE_TREZOR_BASE_H + + +#include <cstddef> +#include <string> +#include "device/device.hpp" +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include <boost/scope_exit.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> +#include "cryptonote_config.h" +#include "trezor.hpp" + +//automatic lock one more level on device ensuring the current thread is allowed to use it +#define AUTO_LOCK_CMD() \ + /* lock both mutexes without deadlock*/ \ + boost::lock(device_locker, command_locker); \ + /* make sure both already-locked mutexes are unlocked at the end of scope */ \ + boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ + boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) + + +namespace hw { +namespace trezor { + +#if WITH_DEVICE_TREZOR + class device_trezor_base; + + /** + * Trezor device callbacks + */ + class trezor_callback { + public: + virtual void on_button_request() {}; + virtual void on_pin_request(epee::wipeable_string & pin) {}; + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {}; + virtual void on_passphrase_state_request(const std::string & state) {}; + }; + + /** + * Default Trezor protocol client callback + */ + class trezor_protocol_callback { + protected: + device_trezor_base & device; + + public: + explicit trezor_protocol_callback(device_trezor_base & device): device(device) {} + + std::shared_ptr<google::protobuf::Message> on_button_request(const messages::common::ButtonRequest * msg); + std::shared_ptr<google::protobuf::Message> on_pin_matrix_request(const messages::common::PinMatrixRequest * msg); + std::shared_ptr<google::protobuf::Message> on_passphrase_request(const messages::common::PassphraseRequest * msg); + std::shared_ptr<google::protobuf::Message> on_passphrase_state_request(const messages::common::PassphraseStateRequest * msg); + + std::shared_ptr<google::protobuf::Message> on_message(const google::protobuf::Message * msg, messages::MessageType message_type){ + MDEBUG("on_general_message"); + return on_message_dispatch(msg, message_type); + } + + std::shared_ptr<google::protobuf::Message> on_message_dispatch(const google::protobuf::Message * msg, messages::MessageType message_type){ + if (message_type == messages::MessageType_ButtonRequest){ + return on_button_request(dynamic_cast<const messages::common::ButtonRequest*>(msg)); + } else if (message_type == messages::MessageType_PassphraseRequest) { + return on_passphrase_request(dynamic_cast<const messages::common::PassphraseRequest*>(msg)); + } else if (message_type == messages::MessageType_PassphraseStateRequest) { + return on_passphrase_state_request(dynamic_cast<const messages::common::PassphraseStateRequest*>(msg)); + } else if (message_type == messages::MessageType_PinMatrixRequest) { + return on_pin_matrix_request(dynamic_cast<const messages::common::PinMatrixRequest*>(msg)); + } else { + return nullptr; + } + } + }; + + /** + * TREZOR device template with basic functions + */ + class device_trezor_base : public hw::core::device_default { + protected: + + // Locker for concurrent access + mutable boost::recursive_mutex device_locker; + mutable boost::mutex command_locker; + + std::shared_ptr<Transport> m_transport; + std::shared_ptr<trezor_protocol_callback> m_protocol_callback; + std::shared_ptr<trezor_callback> m_callback; + + std::string full_name; + + cryptonote::network_type network_type; + + // + // Internal methods + // + + void require_connected(); + void call_ping_unsafe(); + void test_ping(); + + /** + * Client communication wrapper, handles specific Trezor protocol. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template<class t_message> + std::shared_ptr<t_message> + client_exchange(const std::shared_ptr<const google::protobuf::Message> &req, + const boost::optional<messages::MessageType> & resp_type = boost::none, + const boost::optional<std::vector<messages::MessageType>> & resp_types = boost::none, + const boost::optional<messages::MessageType*> & resp_type_ptr = boost::none, + bool open_session = false, + unsigned depth=0) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + const bool accepting_base = boost::is_same<google::protobuf::Message, t_message>::value; + if (resp_types && !accepting_base){ + throw std::invalid_argument("Cannot specify list of accepted types and not using generic response"); + } + + // Open session if required + if (open_session && depth == 0){ + try { + m_transport->open(); + } catch (const std::exception& e) { + std::throw_with_nested(exc::SessionException("Could not open session")); + } + } + + // Scoped session closer + BOOST_SCOPE_EXIT_ALL(&, this) { + if (open_session && depth == 0){ + this->getTransport()->close(); + } + }; + + // Write the request + CHECK_AND_ASSERT_THROW_MES(req, "Request is null"); + this->getTransport()->write(*req); + + // Read the response + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + + // We may have several roundtrips with the handler + this->getTransport()->read(msg_resp, &msg_resp_type); + if (resp_type_ptr){ + *(resp_type_ptr.get()) = msg_resp_type; + } + + // Determine type of expected message response + messages::MessageType required_type = accepting_base ? messages::MessageType_Success : + (resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>()); + + if (msg_resp_type == messages::MessageType_Failure) { + throw_failure_exception(dynamic_cast<messages::common::Failure *>(msg_resp.get())); + + } else if (!accepting_base && msg_resp_type == required_type) { + return message_ptr_retype<t_message>(msg_resp); + + } else { + auto resp = this->getProtocolCallback()->on_message(msg_resp.get(), msg_resp_type); + if (resp) { + return this->client_exchange<t_message>(resp, boost::none, resp_types, resp_type_ptr, false, depth + 1); + + } else if (accepting_base && (!resp_types || + std::find(resp_types.get().begin(), resp_types.get().end(), msg_resp_type) != resp_types.get().end())) { + return message_ptr_retype<t_message>(msg_resp); + + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + } + } + + /** + * Utility method to set address_n and network type to the message requets. + */ + template<class t_message> + void set_msg_addr(t_message * msg, + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Message is null"); + msg->clear_address_n(); + if (path){ + for(auto x : path.get()){ + msg->add_address_n(x); + } + } else { + for (unsigned int i : DEFAULT_BIP44_PATH) { + msg->add_address_n(i); + } + } + + if (network_type){ + msg->set_network_type(static_cast<uint32_t>(network_type.get())); + } else { + msg->set_network_type(static_cast<uint32_t>(this->network_type)); + } + } + + public: + device_trezor_base(); + ~device_trezor_base() override; + + device_trezor_base(const device_trezor_base &device) = delete ; + device_trezor_base& operator=(const device_trezor_base &device) = delete; + + explicit operator bool() const override {return true;} + device_type get_type() const override {return device_type::TREZOR;}; + + bool reset(); + + // Default derivation path for Monero + static const uint32_t DEFAULT_BIP44_PATH[3]; + + std::shared_ptr<Transport> getTransport(){ + return m_transport; + } + + std::shared_ptr<trezor_protocol_callback> getProtocolCallback(){ + return m_protocol_callback; + } + + std::shared_ptr<trezor_callback> getCallback(){ + return m_callback; + } + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + bool set_name(const std::string &name) override; + + const std::string get_name() const override; + bool init() override; + bool release() override; + bool connect() override; + bool disconnect() override; + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + void lock() override; + void unlock() override; + bool try_lock() override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Device ping, no-throw + */ + bool ping(); + + // Protocol callbacks + void on_button_request(); + void on_pin_request(epee::wipeable_string & pin); + void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + void on_passphrase_state_request(const std::string & state); + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_BASE_H diff --git a/src/device_trezor/trezor.hpp b/src/device_trezor/trezor.hpp new file mode 100644 index 000000000..8abdd2c18 --- /dev/null +++ b/src/device_trezor/trezor.hpp @@ -0,0 +1,44 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TREZOR_HPP +#define MONERO_TREZOR_HPP + +#include "trezor/trezor_defs.hpp" + +#ifdef HAVE_PROTOBUF +#include "trezor/transport.hpp" +#include "trezor/messages/messages.pb.h" +#include "trezor/messages/messages-common.pb.h" +#include "trezor/messages/messages-management.pb.h" +#include "trezor/messages/messages-monero.pb.h" +#include "trezor/protocol.hpp" +#endif + +#endif //MONERO_TREZOR_HPP diff --git a/src/device_trezor/trezor/exceptions.hpp b/src/device_trezor/trezor/exceptions.hpp new file mode 100644 index 000000000..197dc43a4 --- /dev/null +++ b/src/device_trezor/trezor/exceptions.hpp @@ -0,0 +1,193 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_EXCEPTIONS_H +#define MONERO_EXCEPTIONS_H + +#include <exception> +#include <string> +#include <boost/optional.hpp> + +namespace hw { +namespace trezor { +namespace exc { + + class SecurityException : public std::exception { + protected: + boost::optional<std::string> reason; + + public: + SecurityException(): reason("General Security exception"){} + explicit SecurityException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class Poly1305TagInvalid: public SecurityException { + public: + using SecurityException::SecurityException; + Poly1305TagInvalid(): SecurityException("Poly1305 authentication tag invalid"){} + }; + + class TrezorException : public std::exception { + protected: + boost::optional<std::string> reason; + + public: + TrezorException(): reason("General Trezor exception"){} + explicit TrezorException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class CommunicationException: public TrezorException { + public: + using TrezorException::TrezorException; + CommunicationException(): TrezorException("Trezor communication error"){} + }; + + class EncodingException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + EncodingException(): CommunicationException("Trezor message encoding error"){} + }; + + class NotConnectedException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + NotConnectedException(): CommunicationException("Trezor not connected"){} + }; + + class DeviceNotResponsiveException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceNotResponsiveException(): CommunicationException("Trezor does not respond to ping"){} + }; + + class DeviceAcquireException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceAcquireException(): CommunicationException("Trezor could not be acquired"){} + }; + + class SessionException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + SessionException(): CommunicationException("Trezor session expired"){} + }; + + class TimeoutException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + TimeoutException(): CommunicationException("Trezor communication timeout"){} + }; + + class ProtocolException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + ProtocolException(): CommunicationException("Trezor protocol error"){} + }; + + // Communication protocol namespace + // Separated to distinguish between client and Trezor side exceptions. +namespace proto { + + class SecurityException : public ProtocolException { + public: + using ProtocolException::ProtocolException; + SecurityException(): ProtocolException("Security assertion violated in the protocol"){} + }; + + class FailureException : public ProtocolException { + private: + boost::optional<uint32_t> code; + boost::optional<std::string> message; + public: + using ProtocolException::ProtocolException; + FailureException(): ProtocolException("Trezor returned failure"){} + FailureException(boost::optional<uint32_t> code, + boost::optional<std::string> message) + : code(code), message(message) { + reason = "Trezor returned failure: code=" + + (code ? std::to_string(code.get()) : "") + + ", message=" + (message ? message.get() : ""); + }; + }; + + class UnexpectedMessageException : public FailureException { + public: + using FailureException::FailureException; + UnexpectedMessageException(): FailureException("Trezor claims unexpected message received"){} + }; + + class CancelledException : public FailureException { + public: + using FailureException::FailureException; + CancelledException(): FailureException("Trezor returned: cancelled operation"){} + }; + + class PinExpectedException : public FailureException { + public: + using FailureException::FailureException; + PinExpectedException(): FailureException("Trezor claims PIN is expected"){} + }; + + class InvalidPinException : public FailureException { + public: + using FailureException::FailureException; + InvalidPinException(): FailureException("Trezor claims PIN is invalid"){} + }; + + class NotEnoughFundsException : public FailureException { + public: + using FailureException::FailureException; + NotEnoughFundsException(): FailureException("Trezor claims not enough funds"){} + }; + + class NotInitializedException : public FailureException { + public: + using FailureException::FailureException; + NotInitializedException(): FailureException("Trezor claims not initialized"){} + }; + + class FirmwareErrorException : public FailureException { + public: + using FailureException::FailureException; + FirmwareErrorException(): FailureException("Trezor returned firmware error"){} + }; + +} +} +} +} +#endif //MONERO_EXCEPTIONS_H diff --git a/src/device_trezor/trezor/messages/.gitignore b/src/device_trezor/trezor/messages/.gitignore new file mode 100644 index 000000000..32f7a77e5 --- /dev/null +++ b/src/device_trezor/trezor/messages/.gitignore @@ -0,0 +1,2 @@ +# protobuf generated code +* diff --git a/src/device_trezor/trezor/messages_map.cpp b/src/device_trezor/trezor/messages_map.cpp new file mode 100644 index 000000000..b0d1aa254 --- /dev/null +++ b/src/device_trezor/trezor/messages_map.cpp @@ -0,0 +1,125 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "messages_map.hpp" +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +using namespace std; +using namespace hw::trezor; + +namespace hw{ +namespace trezor +{ + + const char * TYPE_PREFIX = "MessageType_"; + const char * PACKAGES[] = { + "hw.trezor.messages.", + "hw.trezor.messages.common.", + "hw.trezor.messages.management.", + "hw.trezor.messages.monero." + }; + + google::protobuf::Message * MessageMapper::get_message(int wire_number) { + return MessageMapper::get_message(static_cast<messages::MessageType>(wire_number)); + } + + google::protobuf::Message * MessageMapper::get_message(messages::MessageType wire_number) { + const string &messageTypeName = hw::trezor::messages::MessageType_Name(wire_number); + if (messageTypeName.empty()) { + throw exc::EncodingException(std::string("Message descriptor not found: ") + std::to_string(wire_number)); + } + + string messageName = messageTypeName.substr(strlen(TYPE_PREFIX)); + return MessageMapper::get_message(messageName); + } + + google::protobuf::Message * MessageMapper::get_message(const std::string & msg_name) { + // Each package instantiation so lookup works + hw::trezor::messages::common::Success::default_instance(); + hw::trezor::messages::management::Cancel::default_instance(); + hw::trezor::messages::monero::MoneroGetAddress::default_instance(); + + google::protobuf::Descriptor const * desc = nullptr; + for(const string &text : PACKAGES){ + desc = google::protobuf::DescriptorPool::generated_pool() + ->FindMessageTypeByName(text + msg_name); + if (desc != nullptr){ + break; + } + } + + if (desc == nullptr){ + throw exc::EncodingException(std::string("Message not found: ") + msg_name); + } + + google::protobuf::Message* message = + google::protobuf::MessageFactory::generated_factory() + ->GetPrototype(desc)->New(); + + return message; + +// // CODEGEN way, fast +// switch(wire_number){ +// case 501: +// return new messages::monero::MoneroTransactionSignRequest(); +// default: +// throw std::runtime_error("not implemented"); +// } +// +// // CODEGEN message -> number: specification +// // messages::MessageType get_message_wire_number(const messages::monero::MoneroTransactionSignRequest * msg) { return 501; } +// // messages::MessageType get_message_wire_number(const messages::management::ping * msg) +// + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message * msg){ + return MessageMapper::get_message_wire_number(msg->GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message & msg){ + return MessageMapper::get_message_wire_number(msg.GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const std::string & msg_name){ + string enumMessageName = std::string(TYPE_PREFIX) + msg_name; + + messages::MessageType res; + bool r = hw::trezor::messages::MessageType_Parse(enumMessageName, &res); + if (!r){ + throw exc::EncodingException(std::string("Message ") + msg_name + " not found"); + } + + return res; + } + +} +} diff --git a/src/device_trezor/trezor/messages_map.hpp b/src/device_trezor/trezor/messages_map.hpp new file mode 100644 index 000000000..f61338f09 --- /dev/null +++ b/src/device_trezor/trezor/messages_map.hpp @@ -0,0 +1,94 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_MESSAGES_MAP_H +#define MONERO_MESSAGES_MAP_H + +#include <string> +#include <type_traits> +#include <memory> +#include "exceptions.hpp" + +#include "trezor_defs.hpp" + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/generated_message_util.h> +#include <google/protobuf/repeated_field.h> +#include <google/protobuf/extension_set.h> +#include <google/protobuf/generated_enum_reflection.h> +#include "google/protobuf/descriptor.pb.h" + +#include "messages/messages.pb.h" + +namespace hw { +namespace trezor { + + class MessageMapper{ + public: + MessageMapper() { + + } + + static ::google::protobuf::Message * get_message(int wire_number); + static ::google::protobuf::Message * get_message(messages::MessageType); + static ::google::protobuf::Message * get_message(const std::string & msg_name); + static messages::MessageType get_message_wire_number(const google::protobuf::Message * msg); + static messages::MessageType get_message_wire_number(const google::protobuf::Message & msg); + static messages::MessageType get_message_wire_number(const std::string & msg_name); + + template<class t_message> + static messages::MessageType get_message_wire_number() { + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + return get_message_wire_number(t_message::default_instance().GetDescriptor()->name()); + } + }; + + template<class t_message> + std::shared_ptr<t_message> message_ptr_retype(std::shared_ptr<google::protobuf::Message> & in){ + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + if (!in){ + return nullptr; + } + + return std::dynamic_pointer_cast<t_message>(in); + } + + template<class t_message> + std::shared_ptr<t_message> message_ptr_retype_static(std::shared_ptr<google::protobuf::Message> & in){ + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + if (!in){ + return nullptr; + } + + return std::static_pointer_cast<t_message>(in); + } + +}} + +#endif //MONERO_MESSAGES_MAP_H diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp new file mode 100644 index 000000000..c4a92426c --- /dev/null +++ b/src/device_trezor/trezor/protocol.cpp @@ -0,0 +1,891 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "protocol.hpp" +#include <unordered_map> +#include <set> +#include <utility> +#include <boost/endian/conversion.hpp> +#include <common/apply_permutation.h> +#include <ringct/rctSigs.h> +#include <ringct/bulletproofs.h> +#include "cryptonote_config.h" +#include <sodium.h> +#include <sodium/crypto_verify_32.h> +#include <sodium/crypto_aead_chacha20poly1305.h> + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::ec_scalar & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::hash & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::rct::key & key){ + return std::string(reinterpret_cast<const char*>(key.bytes), sizeof(key.bytes)); + } + + void string_to_key(::crypto::ec_scalar & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::crypto::ec_point & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::rct::key & key, const std::string & str){ + if (str.size() != sizeof(key.bytes)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.bytes)) + " B"); + } + memcpy(key.bytes, str.data(), sizeof(key.bytes)); + } + +namespace crypto { +namespace chacha { + + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext){ + if (length < 16){ + throw std::invalid_argument("Ciphertext length too small"); + } + + unsigned long long int cip_len = length; + auto r = crypto_aead_chacha20poly1305_ietf_decrypt( + reinterpret_cast<unsigned char *>(plaintext), &cip_len, nullptr, + static_cast<const unsigned char *>(ciphertext), length, nullptr, 0, iv, key); + + if (r != 0){ + throw exc::Poly1305TagInvalid(); + } + } + +} +} + + +// Cold Key image sync +namespace ki { + + bool key_image_data(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::vector<MoneroTransferDetails> & res) + { + for(auto & td : transfers){ + ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td); + const std::vector<::crypto::public_key> additional_tx_pub_keys = cryptonote::get_additional_tx_pub_keys_from_extra(td.m_tx); + + res.emplace_back(); + auto & cres = res.back(); + + cres.set_out_key(key_to_string(boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key)); + cres.set_tx_pub_key(key_to_string(tx_pub_key)); + cres.set_internal_output_index(td.m_internal_output_index); + for(auto & aux : additional_tx_pub_keys){ + cres.add_additional_tx_pub_keys(key_to_string(aux)); + } + } + + return true; + } + + std::string compute_hash(const MoneroTransferDetails & rr){ + KECCAK_CTX kck; + uint8_t md[32]; + + CHECK_AND_ASSERT_THROW_MES(rr.out_key().size() == 32, "Invalid out_key size"); + CHECK_AND_ASSERT_THROW_MES(rr.tx_pub_key().size() == 32, "Invalid tx_pub_key size"); + + keccak_init(&kck); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(rr.out_key().data()), 32); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(rr.tx_pub_key().data()), 32); + for (const auto &aux : rr.additional_tx_pub_keys()){ + CHECK_AND_ASSERT_THROW_MES(aux.size() == 32, "Invalid aux size"); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(aux.data()), 32); + } + + auto index_serialized = tools::get_varint_data(rr.internal_output_index()); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(index_serialized.data()), index_serialized.size()); + keccak_finish(&kck, md); + return std::string(reinterpret_cast<const char*>(md), sizeof(md)); + } + + void generate_commitment(std::vector<MoneroTransferDetails> & mtds, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req) + { + req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>(); + + KECCAK_CTX kck; + uint8_t final_hash[32]; + keccak_init(&kck); + + for(auto &cur : mtds){ + auto hash = compute_hash(cur); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(hash.data()), hash.size()); + } + keccak_finish(&kck, final_hash); + + req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>(); + req->set_hash(std::string(reinterpret_cast<const char*>(final_hash), 32)); + req->set_num(transfers.size()); + + std::unordered_map<uint32_t, std::set<uint32_t>> sub_indices; + for (auto &cur : transfers){ + auto search = sub_indices.emplace(cur.m_subaddr_index.major, std::set<uint32_t>()); + auto & st = search.first->second; + st.insert(cur.m_subaddr_index.minor); + } + + for (auto& x: sub_indices){ + auto subs = req->add_subs(); + subs->set_account(x.first); + for(auto minor : x.second){ + subs->add_minor_indices(minor); + } + } + } + +} + +// Cold transaction signing +namespace tx { + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src){ + dst->set_view_public_key(key_to_string(src->m_view_public_key)); + dst->set_spend_public_key(key_to_string(src->m_spend_public_key)); + } + + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){ + dst->set_amount(src->amount); + dst->set_is_subaddress(src->is_subaddress); + translate_address(dst->mutable_addr(), &(src->addr)); + } + + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src){ + for(auto & cur : src->outputs){ + auto out = dst->add_outputs(); + out->set_idx(cur.first); + translate_rct_key(out->mutable_key(), &(cur.second)); + } + + dst->set_real_output(src->real_output); + dst->set_real_out_tx_key(key_to_string(src->real_out_tx_key)); + for(auto & cur : src->real_out_additional_tx_keys){ + dst->add_real_out_additional_tx_keys(key_to_string(cur)); + } + + dst->set_real_output_in_tx_index(src->real_output_in_tx_index); + dst->set_amount(src->amount); + dst->set_rct(src->rct); + dst->set_mask(key_to_string(src->mask)); + translate_klrki(dst->mutable_multisig_klrki(), &(src->multisig_kLRki)); + } + + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src){ + dst->set_k(key_to_string(src->k)); + dst->set_l(key_to_string(src->L)); + dst->set_r(key_to_string(src->R)); + dst->set_ki(key_to_string(src->ki)); + } + + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src){ + dst->set_dest(key_to_string(src->dest)); + dst->set_commitment(key_to_string(src->mask)); + } + + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + return hash_addr(addr->spend_public_key(), addr->view_public_key(), amount, is_subaddr); + } + + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + ::crypto::public_key spend{}, view{}; + if (spend_key.size() != 32 || view_key.size() != 32){ + throw std::invalid_argument("Public keys have invalid sizes"); + } + + memcpy(spend.data, spend_key.data(), 32); + memcpy(view.data, view_key.data(), 32); + return hash_addr(&spend, &view, amount, is_subaddr); + } + + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + char buff[64+8+1]; + size_t offset = 0; + + memcpy(buff + offset, spend_key->data, 32); offset += 32; + memcpy(buff + offset, view_key->data, 32); offset += 32; + + if (amount){ + memcpy(buff + offset, (uint8_t*) &(amount.get()), sizeof(amount.get())); offset += sizeof(amount.get()); + } + + if (is_subaddr){ + buff[offset] = is_subaddr.get(); + offset += 1; + } + + return std::string(buff, offset); + } + + TData::TData() { + in_memory = false; + rsig_type = 0; + cur_input_idx = 0; + cur_output_idx = 0; + cur_batch_idx = 0; + cur_output_in_batch_idx = 0; + } + + Signer::Signer(wallet_shim *wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx, hw::tx_aux_data * aux_data) { + m_wallet2 = wallet2; + m_unsigned_tx = unsigned_tx; + m_aux_data = aux_data; + m_tx_idx = tx_idx; + m_ct.tx_data = cur_tx(); + m_multisig = false; + } + + void Signer::extract_payment_id(){ + const std::vector<uint8_t>& tx_extra = cur_tx().extra; + m_ct.tsx_data.set_payment_id(""); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + cryptonote::parse_tx_extra(tx_extra, tx_extra_fields); // ok if partially parsed + cryptonote::tx_extra_nonce extra_nonce; + + ::crypto::hash payment_id{}; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + ::crypto::hash8 payment_id8{}; + if(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id8.data, 8)); + } + else if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id.data, 32)); + } + } + } + + static unsigned get_rsig_type(bool use_bulletproof, size_t num_outputs){ + if (!use_bulletproof){ + return rct::RangeProofBorromean; + } else if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + return rct::RangeProofMultiOutputBulletproof; + } else { + return rct::RangeProofPaddedBulletproof; + } + } + + static void generate_rsig_batch_sizes(std::vector<uint64_t> &batches, unsigned rsig_type, size_t num_outputs){ + size_t amount_batched = 0; + + while(amount_batched < num_outputs){ + if (rsig_type == rct::RangeProofBorromean || rsig_type == rct::RangeProofBulletproof) { + batches.push_back(1); + amount_batched += 1; + + } else if (rsig_type == rct::RangeProofPaddedBulletproof){ + if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + throw std::invalid_argument("BP padded can support only BULLETPROOF_MAX_OUTPUTS statements"); + } + batches.push_back(num_outputs); + amount_batched += num_outputs; + + } else if (rsig_type == rct::RangeProofMultiOutputBulletproof){ + size_t batch_size = 1; + while (batch_size * 2 + amount_batched <= num_outputs && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS){ + batch_size *= 2; + } + batch_size = std::min(batch_size, num_outputs - amount_batched); + batches.push_back(batch_size); + amount_batched += batch_size; + + } else { + throw std::invalid_argument("Unknown rsig type"); + } + } + } + + void Signer::compute_integrated_indices(TsxData * tsx_data){ + if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ + return; + } + + auto & chg = tsx_data->change_dts(); + std::string change_hash = hash_addr(&chg.addr(), chg.amount(), chg.is_subaddress()); + + std::vector<uint32_t> integrated_indices; + std::set<std::string> integrated_hashes; + for (auto & cur : m_aux_data->tx_recipients){ + if (!cur.has_payment_id){ + continue; + } + integrated_hashes.emplace(hash_addr(&cur.address.m_spend_public_key, &cur.address.m_view_public_key)); + } + + ssize_t idx = -1; + for (auto & cur : tsx_data->outputs()){ + idx += 1; + + std::string c_hash = hash_addr(&cur.addr(), cur.amount(), cur.is_subaddress()); + if (c_hash == change_hash || cur.is_subaddress()){ + continue; + } + + c_hash = hash_addr(&cur.addr()); + if (integrated_hashes.find(c_hash) != integrated_hashes.end()){ + integrated_indices.push_back((uint32_t)idx); + } + } + + if (!integrated_indices.empty()){ + assign_to_repeatable(tsx_data->mutable_integrated_indices(), integrated_indices.begin(), integrated_indices.end()); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionInitRequest> Signer::step_init(){ + // extract payment ID from construction data + auto & tsx_data = m_ct.tsx_data; + auto & tx = cur_tx(); + + m_ct.tx.version = 2; + m_ct.tx.unlock_time = tx.unlock_time; + + tsx_data.set_version(1); + tsx_data.set_unlock_time(tx.unlock_time); + tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size())); + tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1)); + tsx_data.set_account(tx.subaddr_account); + assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end()); + + // Rsig decision + auto rsig_data = tsx_data.mutable_rsig_data(); + m_ct.rsig_type = get_rsig_type(tx.use_bulletproofs, tx.splitted_dsts.size()); + rsig_data->set_rsig_type(m_ct.rsig_type); + + generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); + assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); + + translate_dst_entry(tsx_data.mutable_change_dts(), &(tx.change_dts)); + for(auto & cur : tx.splitted_dsts){ + auto dst = tsx_data.mutable_outputs()->Add(); + translate_dst_entry(dst, &cur); + } + + compute_integrated_indices(&tsx_data); + + int64_t fee = 0; + for(auto & cur_in : tx.sources){ + fee += cur_in.amount; + } + for(auto & cur_out : tx.splitted_dsts){ + fee -= cur_out.amount; + } + if (fee < 0){ + throw std::invalid_argument("Fee cannot be negative"); + } + + tsx_data.set_fee(static_cast<google::protobuf::uint64>(fee)); + this->extract_payment_id(); + + auto init_req = std::make_shared<messages::monero::MoneroTransactionInitRequest>(); + init_req->set_version(0); + init_req->mutable_tsx_data()->CopyFrom(tsx_data); + return init_req; + } + + void Signer::step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack){ + m_ct.in_memory = false; + if (ack->has_rsig_data()){ + m_ct.rsig_param = std::make_shared<MoneroRsigData>(ack->rsig_data()); + } + + assign_from_repeatable(&(m_ct.tx_out_entr_hmacs), ack->hmacs().begin(), ack->hmacs().end()); + } + + std::shared_ptr<messages::monero::MoneroTransactionSetInputRequest> Signer::step_set_input(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < cur_tx().sources.size(), "Invalid source index"); + m_ct.cur_input_idx = idx; + auto res = std::make_shared<messages::monero::MoneroTransactionSetInputRequest>(); + translate_src_entry(res->mutable_src_entr(), &(cur_tx().sources[idx])); + return res; + } + + void Signer::step_set_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetInputAck> ack){ + auto & vini_str = ack->vini(); + + cryptonote::txin_v vini; + if (!cn_deserialize(vini_str.data(), vini_str.size(), vini)){ + throw exc::ProtocolException("Cannot deserialize vin[i]"); + } + + m_ct.tx.vin.emplace_back(vini); + m_ct.tx_in_hmacs.push_back(ack->vini_hmac()); + m_ct.pseudo_outs.push_back(ack->pseudo_out()); + m_ct.pseudo_outs_hmac.push_back(ack->pseudo_out_hmac()); + m_ct.alphas.push_back(ack->pseudo_out_alpha()); + m_ct.spend_encs.push_back(ack->spend_key()); + } + + void Signer::sort_ki(){ + const size_t input_size = cur_tx().sources.size(); + + m_ct.source_permutation.clear(); + for (size_t n = 0; n < input_size; ++n){ + m_ct.source_permutation.push_back(n); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx.vin.size() == input_size, "Invalid vector size"); + std::sort(m_ct.source_permutation.begin(), m_ct.source_permutation.end(), [&](const size_t i0, const size_t i1) { + const cryptonote::txin_to_key &tk0 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i0]); + const cryptonote::txin_to_key &tk1 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i1]); + return memcmp(&tk0.k_image, &tk1.k_image, sizeof(tk0.k_image)) > 0; + }); + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_in_hmacs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs_hmac.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.alphas.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.spend_encs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_data.sources.size() == input_size, "Invalid vector size"); + + tools::apply_permutation(m_ct.source_permutation, [&](size_t i0, size_t i1){ + std::swap(m_ct.tx.vin[i0], m_ct.tx.vin[i1]); + std::swap(m_ct.tx_in_hmacs[i0], m_ct.tx_in_hmacs[i1]); + std::swap(m_ct.pseudo_outs[i0], m_ct.pseudo_outs[i1]); + std::swap(m_ct.pseudo_outs_hmac[i0], m_ct.pseudo_outs_hmac[i1]); + std::swap(m_ct.alphas[i0], m_ct.alphas[i1]); + std::swap(m_ct.spend_encs[i0], m_ct.spend_encs[i1]); + std::swap(m_ct.tx_data.sources[i0], m_ct.tx_data.sources[i1]); + }); + } + + std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){ + sort_ki(); + + if (in_memory()){ + return nullptr; + } + + auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>(); + assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); + + return res; + } + + void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){ + if (in_memory()){ + return; + } + } + + std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){ + if (in_memory()){ + return nullptr; + } + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + + m_ct.cur_input_idx = idx; + auto tx = m_ct.tx_data; + auto res = std::make_shared<messages::monero::MoneroTransactionInputViniRequest>(); + auto & vini = m_ct.tx.vin[idx]; + translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + if (!in_memory()) { + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + } + + return res; + } + + void Signer::step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack){ + if (in_memory()){ + return; + } + } + + std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> Signer::step_all_inputs_set(){ + return std::make_shared<messages::monero::MoneroTransactionAllInputsSetRequest>(); + } + + void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){ + if (is_offloading()){ + // If offloading, expect rsig configuration. + if (!ack->has_rsig_data()){ + throw exc::ProtocolException("Rsig offloading requires rsig param"); + } + + auto & rsig_data = ack->rsig_data(); + if (!rsig_data.has_mask()){ + throw exc::ProtocolException("Gamma masks not present in offloaded version"); + } + + auto & mask = rsig_data.mask(); + if (mask.size() != 32 * num_outputs()){ + throw exc::ProtocolException("Invalid number of gamma masks"); + } + + m_ct.rsig_gamma.reserve(num_outputs()); + for(size_t c=0; c < num_outputs(); ++c){ + rct::key cmask{}; + memcpy(cmask.bytes, mask.data() + c * 32, 32); + m_ct.rsig_gamma.emplace_back(cmask); + } + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index"); + + m_ct.cur_output_idx = idx; + m_ct.cur_output_in_batch_idx += 1; // assumes sequential call to step_set_output() + + auto res = std::make_shared<messages::monero::MoneroTransactionSetOutputRequest>(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + // Range sig offloading to the host + if (!is_offloading()) { + return res; + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + if (m_ct.grouping_vct[m_ct.cur_batch_idx] > m_ct.cur_output_in_batch_idx) { + return res; + } + + auto rsig_data = res->mutable_rsig_data(); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + + if (!is_req_bulletproof()){ + if (batch_size > 1){ + throw std::invalid_argument("Borromean cannot batch outputs"); + } + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.rsig_gamma.size(), "Invalid gamma index"); + rct::key C{}, mask = m_ct.rsig_gamma[idx]; + auto genRsig = rct::proveRange(C, mask, cur_dst.amount); // TODO: rsig with given mask + auto serRsig = cn_serialize(genRsig); + m_ct.tx_out_rsigs.emplace_back(genRsig); + rsig_data->set_rsig(serRsig); + + } else { + std::vector<uint64_t> amounts; + rct::keyV masks; + CHECK_AND_ASSERT_THROW_MES(idx + 1 >= batch_size, "Invalid index for batching"); + + for(size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); + + amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); + masks.push_back(m_ct.rsig_gamma[bidx]); + } + + auto bp = bulletproof_PROVE(amounts, masks); + auto serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + rsig_data->set_rsig(serRsig); + } + + return res; + } + + void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ + cryptonote::tx_out tx_out; + rct::rangeSig range_sig{}; + rct::Bulletproof bproof{}; + rct::ctkey out_pk{}; + rct::ecdhTuple ecdh{}; + + bool has_rsig = false; + std::string rsig_buff; + + if (ack->has_rsig_data()){ + auto & rsig_data = ack->rsig_data(); + + if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){ + has_rsig = true; + rsig_buff = rsig_data.rsig(); + + } else if (rsig_data.rsig_parts_size() > 0){ + has_rsig = true; + for (const auto &it : rsig_data.rsig_parts()) { + rsig_buff += it; + } + } + } + + if (!cn_deserialize(ack->tx_out(), tx_out)){ + throw exc::ProtocolException("Cannot deserialize vout[i]"); + } + + if (!cn_deserialize(ack->out_pk(), out_pk)){ + throw exc::ProtocolException("Cannot deserialize out_pk"); + } + + if (!cn_deserialize(ack->ecdh_info(), ecdh)){ + throw exc::ProtocolException("Cannot deserialize ecdhtuple"); + } + + if (has_rsig && !is_req_bulletproof() && !cn_deserialize(rsig_buff, range_sig)){ + throw exc::ProtocolException("Cannot deserialize rangesig"); + } + + if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ + throw exc::ProtocolException("Cannot deserialize bulletproof rangesig"); + } + + m_ct.tx.vout.emplace_back(tx_out); + m_ct.tx_out_hmacs.push_back(ack->vouti_hmac()); + m_ct.tx_out_pk.emplace_back(out_pk); + m_ct.tx_out_ecdh.emplace_back(ecdh); + + if (!has_rsig){ + return; + } + + if (is_req_bulletproof()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + for (size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + + rct::key commitment = m_ct.tx_out_pk[bidx].mask; + commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); + bproof.V.push_back(commitment); + } + + m_ct.tx_out_rsigs.emplace_back(bproof); + if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + + } else { + m_ct.tx_out_rsigs.emplace_back(range_sig); + + if (!rct::verRange(out_pk.mask, boost::get<rct::rangeSig>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } + + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } + + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> Signer::step_all_outs_set(){ + return std::make_shared<messages::monero::MoneroTransactionAllOutSetRequest>(); + } + + void Signer::step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev){ + m_ct.rv = std::make_shared<rct::rctSig>(); + m_ct.rv->txnFee = ack->rv().txn_fee(); + m_ct.rv->type = static_cast<uint8_t>(ack->rv().rv_type()); + string_to_key(m_ct.rv->message, ack->rv().message()); + + // Extra copy + m_ct.tx.extra.clear(); + auto extra = ack->extra(); + auto extra_data = extra.data(); + m_ct.tx.extra.reserve(extra.size()); + for(size_t i = 0; i < extra.size(); ++i){ + m_ct.tx.extra.push_back(static_cast<uint8_t>(extra_data[i])); + } + + ::crypto::hash tx_prefix_hash{}; + cryptonote::get_transaction_prefix_hash(m_ct.tx, tx_prefix_hash); + m_ct.tx_prefix_hash = key_to_string(tx_prefix_hash); + if (crypto_verify_32(reinterpret_cast<const unsigned char *>(tx_prefix_hash.data), + reinterpret_cast<const unsigned char *>(ack->tx_prefix_hash().data()))){ + throw exc::proto::SecurityException("Transaction prefix has does not match to the computed value"); + } + + // RctSig + auto num_sources = m_ct.tx_data.sources.size(); + if (is_simple() || is_req_bulletproof()){ + auto dst = &m_ct.rv->pseudoOuts; + if (is_bulletproof()){ + dst = &m_ct.rv->p.pseudoOuts; + } + + dst->clear(); + for (const auto &pseudo_out : m_ct.pseudo_outs) { + dst->emplace_back(); + string_to_key(dst->back(), pseudo_out); + } + + m_ct.rv->mixRing.resize(num_sources); + } else { + m_ct.rv->mixRing.resize(m_ct.tsx_data.mixin()); + m_ct.rv->mixRing[0].resize(num_sources); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_out_pk.size() == m_ct.tx_out_ecdh.size(), "Invalid vector sizes"); + for(size_t i = 0; i < m_ct.tx_out_ecdh.size(); ++i){ + m_ct.rv->outPk.push_back(m_ct.tx_out_pk[i]); + m_ct.rv->ecdhInfo.push_back(m_ct.tx_out_ecdh[i]); + } + + for(size_t i = 0; i < m_ct.tx_out_rsigs.size(); ++i){ + if (is_bulletproof()){ + m_ct.rv->p.bulletproofs.push_back(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs[i])); + } else { + m_ct.rv->p.rangeSigs.push_back(boost::get<rct::rangeSig>(m_ct.tx_out_rsigs[i])); + } + } + + rct::key hash_computed = rct::get_pre_mlsag_hash(*(m_ct.rv), hwdev); + auto & hash = ack->full_message_hash(); + + if (hash.size() != 32){ + throw exc::ProtocolException("Returned mlsag hash has invalid size"); + } + + if (crypto_verify_32(reinterpret_cast<const unsigned char *>(hash_computed.bytes), + reinterpret_cast<const unsigned char *>(hash.data()))){ + throw exc::proto::SecurityException("Computed MLSAG does not match"); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSignInputRequest> Signer::step_sign_input(size_t idx){ + m_ct.cur_input_idx = idx; + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.alphas.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.spend_encs.size(), "Invalid transaction index"); + + auto res = std::make_shared<messages::monero::MoneroTransactionSignInputRequest>(); + translate_src_entry(res->mutable_src_entr(), &(m_ct.tx_data.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(m_ct.tx.vin[idx])); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + res->set_pseudo_out_alpha(m_ct.alphas[idx]); + res->set_spend_key(m_ct.spend_encs[idx]); + if (!in_memory()){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + } + return res; + } + + void Signer::step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack){ + rct::mgSig mg; + if (!cn_deserialize(ack->signature(), mg)){ + throw exc::ProtocolException("Cannot deserialize mg[i]"); + } + + m_ct.rv->p.MGs.push_back(mg); + } + + std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> Signer::step_final(){ + m_ct.tx.rct_signatures = *(m_ct.rv); + return std::make_shared<messages::monero::MoneroTransactionFinalRequest>(); + } + + void Signer::step_final_ack(std::shared_ptr<const messages::monero::MoneroTransactionFinalAck> ack){ + if (m_multisig){ + auto & cout_key = ack->cout_key(); + for(auto & cur : m_ct.couts){ + if (cur.size() != 12 + 32){ + throw std::invalid_argument("Encrypted cout has invalid length"); + } + + char buff[32]; + auto data = cur.data(); + + crypto::chacha::decrypt(data + 12, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); + m_ct.couts_dec.emplace_back(buff, 32); + } + } + + m_ct.enc_salt1 = ack->salt(); + m_ct.enc_salt2 = ack->rand_mult(); + m_ct.enc_keys = ack->tx_enc_keys(); + } + + std::string Signer::store_tx_aux_info(){ + rapidjson::StringBuffer sb; + rapidjson::Writer<rapidjson::StringBuffer> writer(sb); + + rapidjson::Document json; + json.SetObject(); + + rapidjson::Value valueS(rapidjson::kStringType); + rapidjson::Value valueI(rapidjson::kNumberType); + + valueI.SetInt(1); + json.AddMember("version", valueI, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt1.c_str(), m_ct.enc_salt1.size()); + json.AddMember("salt1", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt2.c_str(), m_ct.enc_salt2.size()); + json.AddMember("salt2", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_keys.c_str(), m_ct.enc_keys.size()); + json.AddMember("enc_keys", valueS, json.GetAllocator()); + + json.Accept(writer); + return sb.GetString(); + } + + +} +} +} +} diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp new file mode 100644 index 000000000..99211efed --- /dev/null +++ b/src/device_trezor/trezor/protocol.hpp @@ -0,0 +1,300 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_PROTOCOL_H +#define MONERO_PROTOCOL_H + +#include "trezor_defs.hpp" +#include "device/device_cold.hpp" +#include "messages_map.hpp" +#include "transport.hpp" +#include "wallet/wallet2.h" + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key); + std::string key_to_string(const ::crypto::ec_scalar & key); + std::string key_to_string(const ::crypto::hash & key); + std::string key_to_string(const ::rct::key & key); + + void string_to_key(::crypto::ec_scalar & key, const std::string & str); + void string_to_key(::crypto::ec_point & key, const std::string & str); + void string_to_key(::rct::key & key, const std::string & str); + + template<class sub_t, class InputIterator> + void assign_to_repeatable(::google::protobuf::RepeatedField<sub_t> * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + auto s = dst->Add(); + *s = *it; + } + } + + template<class sub_t, class InputIterator> + void assign_from_repeatable(std::vector<sub_t> * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + dst->push_back(*it); + } + }; + + template<typename T> + bool cn_deserialize(const void * buff, size_t len, T & dst){ + std::stringstream ss; + ss.write(static_cast<const char *>(buff), len); //ss << tx_blob; + binary_archive<false> ba(ss); + bool r = ::serialization::serialize(ba, dst); + return r; + } + + template<typename T> + bool cn_deserialize(const std::string & str, T & dst){ + return cn_deserialize(str.data(), str.size(), dst); + } + + template<typename T> + std::string cn_serialize(T & obj){ + std::ostringstream oss; + binary_archive<true> oar(oss); + bool success = ::serialization::serialize(oar, obj); + if (!success){ + throw exc::EncodingException("Could not CN serialize given object"); + } + return oss.str(); + } + +// Crypto / encryption +namespace crypto { +namespace chacha { + + /** + * Chacha20Poly1305 decryption with tag verification. RFC 7539. + */ + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext); + +} +} + + +// Cold Key image sync +namespace ki { + + using MoneroTransferDetails = messages::monero::MoneroKeyImageSyncStepRequest_MoneroTransferDetails; + using MoneroSubAddressIndicesList = messages::monero::MoneroKeyImageExportInitRequest_MoneroSubAddressIndicesList; + using MoneroExportedKeyImage = messages::monero::MoneroKeyImageSyncStepAck_MoneroExportedKeyImage; + using exported_key_image = hw::device_cold::exported_key_image; + + /** + * Converts transfer details to the MoneroTransferDetails required for KI sync + */ + bool key_image_data(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::vector<MoneroTransferDetails> & res); + + /** + * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync. + */ + std::string compute_hash(const MoneroTransferDetails & rr); + + /** + * Generates KI sync request with commitments computed. + */ + void generate_commitment(std::vector<MoneroTransferDetails> & mtds, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req); + +} + +// Cold transaction signing +namespace tx { + using TsxData = messages::monero::MoneroTransactionInitRequest_MoneroTransactionData; + using MoneroTransactionDestinationEntry = messages::monero::MoneroTransactionDestinationEntry; + using MoneroAccountPublicAddress = messages::monero::MoneroTransactionDestinationEntry_MoneroAccountPublicAddress; + using MoneroTransactionSourceEntry = messages::monero::MoneroTransactionSourceEntry; + using MoneroMultisigKLRki = messages::monero::MoneroTransactionSourceEntry_MoneroMultisigKLRki; + using MoneroOutputEntry = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry; + using MoneroRctKey = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry_MoneroRctKeyPublic; + using MoneroRsigData = messages::monero::MoneroTransactionRsigData; + + using tx_construction_data = tools::wallet2::tx_construction_data; + using unsigned_tx_set = tools::wallet2::unsigned_tx_set; + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src); + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src); + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src); + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src); + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src); + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + + typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v; + + /** + * Transaction signer state holder. + */ + class TData { + public: + TsxData tsx_data; + tx_construction_data tx_data; + cryptonote::transaction tx; + bool in_memory; + unsigned rsig_type; + std::vector<uint64_t> grouping_vct; + std::shared_ptr<MoneroRsigData> rsig_param; + size_t cur_input_idx; + size_t cur_output_idx; + size_t cur_batch_idx; + size_t cur_output_in_batch_idx; + + std::vector<std::string> tx_in_hmacs; + std::vector<std::string> tx_out_entr_hmacs; + std::vector<std::string> tx_out_hmacs; + std::vector<rsig_v> tx_out_rsigs; + std::vector<rct::ctkey> tx_out_pk; + std::vector<rct::ecdhTuple> tx_out_ecdh; + std::vector<size_t> source_permutation; + std::vector<std::string> alphas; + std::vector<std::string> spend_encs; + std::vector<std::string> pseudo_outs; + std::vector<std::string> pseudo_outs_hmac; + std::vector<std::string> couts; + std::vector<std::string> couts_dec; + std::vector<rct::key> rsig_gamma; + std::string tx_prefix_hash; + std::string enc_salt1; + std::string enc_salt2; + std::string enc_keys; + + std::shared_ptr<rct::rctSig> rv; + + TData(); + }; + + class Signer { + private: + TData m_ct; + wallet_shim * m_wallet2; + + size_t m_tx_idx; + const unsigned_tx_set * m_unsigned_tx; + hw::tx_aux_data * m_aux_data; + + bool m_multisig; + + const tx_construction_data & cur_tx(){ + CHECK_AND_ASSERT_THROW_MES(m_tx_idx < m_unsigned_tx->txes.size(), "Invalid transaction index"); + return m_unsigned_tx->txes[m_tx_idx]; + } + + void extract_payment_id(); + void compute_integrated_indices(TsxData * tsx_data); + + public: + Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); + + std::shared_ptr<messages::monero::MoneroTransactionInitRequest> step_init(); + void step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionSetInputRequest> step_set_input(size_t idx); + void step_set_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetInputAck> ack); + + void sort_ki(); + std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> step_permutation(); + void step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> step_set_vini_input(size_t idx); + void step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> step_all_inputs_set(); + void step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_set_output(size_t idx); + void step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> step_all_outs_set(); + void step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev); + + std::shared_ptr<messages::monero::MoneroTransactionSignInputRequest> step_sign_input(size_t idx); + void step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> step_final(); + void step_final_ack(std::shared_ptr<const messages::monero::MoneroTransactionFinalAck> ack); + + std::string store_tx_aux_info(); + + bool in_memory() const { + return m_ct.in_memory; + } + + bool is_simple() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeSimple; + } + + bool is_req_bulletproof() const { + return m_ct.tx_data.use_bulletproofs; + } + + bool is_bulletproof() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeBulletproof; + } + + bool is_offloading() const { + return m_ct.rsig_param && m_ct.rsig_param->offload_type() != 0; + } + + size_t num_outputs() const { + return m_ct.tx_data.splitted_dsts.size(); + } + + size_t num_inputs() const { + return m_ct.tx_data.sources.size(); + } + + const TData & tdata() const { + return m_ct; + } + }; + +} + +} +} +} + + +#endif //MONERO_PROTOCOL_H diff --git a/src/device_trezor/trezor/tools/README.md b/src/device_trezor/trezor/tools/README.md new file mode 100644 index 000000000..91a8fb3f0 --- /dev/null +++ b/src/device_trezor/trezor/tools/README.md @@ -0,0 +1,36 @@ +# Trezor + +## Messages rebuild + +Install `protoc` for your distribution. + +- `protobuf-compiler` +- `libprotobuf-dev` +- `libprotoc-dev` +- `python-protobuf` + +Python 3 is required. If you don't have python 3 quite an easy way is +to use [pyenv]. + +It is also advised to create own python virtual environment so dependencies +are installed in this project-related virtual environment. + +```bash +python -m venv / +``` + +Make sure your python has `protobuf` package installed + +```bash +pip install protobuf +``` + +Regenerate messages: + +``` +./venv/bin/python3 src/device_trezor/trezor/tools/build_protob.py +``` + +The messages regeneration is done also automatically via cmake. + +[pyenv]: https://github.com/pyenv/pyenv diff --git a/src/device_trezor/trezor/tools/build_protob.py b/src/device_trezor/trezor/tools/build_protob.py new file mode 100644 index 000000000..2611f3296 --- /dev/null +++ b/src/device_trezor/trezor/tools/build_protob.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +import os +import subprocess +import sys + +CWD = os.path.dirname(os.path.realpath(__file__)) +ROOT_DIR = os.path.abspath(os.path.join(CWD, "..", "..", "..", "..")) +TREZOR_COMMON = os.path.join(ROOT_DIR, "external", "trezor-common") +TREZOR_MESSAGES = os.path.join(CWD, "..", "messages") + +# check for existence of the submodule directory +common_defs = os.path.join(TREZOR_COMMON, "defs") +if not os.path.exists(common_defs): + raise ValueError( + "trezor-common submodule seems to be missing.\n" + + 'Use "git submodule update --init --recursive" to retrieve it.' + ) + +# regenerate messages +try: + selected = [ + "messages.proto", + "messages-common.proto", + "messages-management.proto", + "messages-monero.proto", + ] + proto_srcs = [os.path.join(TREZOR_COMMON, "protob", x) for x in selected] + exec_args = [ + sys.executable, + os.path.join(CWD, "pb2cpp.py"), + "-o", + TREZOR_MESSAGES, + ] + proto_srcs + + subprocess.check_call(exec_args) + +except Exception as e: + raise diff --git a/src/device_trezor/trezor/tools/pb2cpp.py b/src/device_trezor/trezor/tools/pb2cpp.py new file mode 100644 index 000000000..eaa8a90ed --- /dev/null +++ b/src/device_trezor/trezor/tools/pb2cpp.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# Converts Google's protobuf python definitions of TREZOR wire messages +# to plain-python objects as used in TREZOR Core and python-trezor + +import argparse +import logging +import os +import re +import shutil +import subprocess +import sys +import glob +import tempfile +import hashlib + + +AUTO_HEADER = "# Automatically generated by pb2cpp\n" + +# Fixing GCC7 compilation error +UNDEF_STATEMENT = """ +#ifdef minor +#undef minor +#endif +""" + + +def which(pgm): + path = os.getenv('PATH') + for p in path.split(os.path.pathsep): + p = os.path.join(p, pgm) + if os.path.exists(p) and os.access(p, os.X_OK): + return p + + +PROTOC = which("protoc") +if not PROTOC: + print("protoc command not found") + sys.exit(1) + +PROTOC_PREFIX = os.path.dirname(os.path.dirname(PROTOC)) +PROTOC_INCLUDE = os.path.join(PROTOC_PREFIX, "include") + + +def namespace_file(fpath, package): + """Adds / replaces package name. Simple regex parsing, may use https://github.com/ph4r05/plyprotobuf later""" + with open(fpath) as fh: + fdata = fh.read() + + re_syntax = re.compile(r"^syntax\s*=") + re_package = re.compile(r"^package\s+([^;]+?)\s*;\s*$") + lines = fdata.split("\n") + + line_syntax = None + line_package = None + for idx, line in enumerate(lines): + if line_syntax is None and re_syntax.match(line): + line_syntax = idx + if line_package is None and re_package.match(line): + line_package = idx + + if package is None: + if line_package is None: + return + else: + lines.pop(line_package) + + else: + new_package = "package %s;" % package + if line_package is None: + lines.insert(line_syntax + 1 if line_syntax is not None else 0, new_package) + else: + lines[line_package] = new_package + + new_fdat = "\n".join(lines) + with open(fpath, "w+") as fh: + fh.write(new_fdat) + return new_fdat + + +def protoc(files, out_dir, additional_includes=(), package=None, force=False): + """Compile code with protoc and return the data.""" + + include_dirs = set() + include_dirs.add(PROTOC_INCLUDE) + include_dirs.update(additional_includes) + + with tempfile.TemporaryDirectory() as tmpdir_protob, tempfile.TemporaryDirectory() as tmpdir_out: + include_dirs.add(tmpdir_protob) + + new_files = [] + for file in files: + bname = os.path.basename(file) + tmp_file = os.path.join(tmpdir_protob, bname) + + shutil.copy(file, tmp_file) + if package is not None: + namespace_file(tmp_file, package) + new_files.append(tmp_file) + + protoc_includes = ["-I" + dir for dir in include_dirs if dir] + + exec_args = ( + [ + PROTOC, + "--cpp_out", + tmpdir_out, + ] + + protoc_includes + + new_files + ) + + subprocess.check_call(exec_args) + + # Fixing gcc compilation and clashes with "minor" field name + add_undef(tmpdir_out) + + # Scan output dir, check file differences + update_message_files(tmpdir_out, out_dir, force) + + +def update_message_files(tmpdir_out, out_dir, force=False): + files = glob.glob(os.path.join(tmpdir_out, '*.pb.*')) + for fname in files: + bname = os.path.basename(fname) + dest_file = os.path.join(out_dir, bname) + if not force and os.path.exists(dest_file): + data = open(fname, 'rb').read() + data_hash = hashlib.sha3_256(data).digest() + data_dest = open(dest_file, 'rb').read() + data_dest_hash = hashlib.sha3_256(data_dest).digest() + if data_hash == data_dest_hash: + continue + + shutil.copy(fname, dest_file) + + +def add_undef(out_dir): + files = glob.glob(os.path.join(out_dir, '*.pb.*')) + for fname in files: + with open(fname) as fh: + lines = fh.readlines() + + idx_insertion = None + for idx in range(len(lines)): + if '@@protoc_insertion_point(includes)' in lines[idx]: + idx_insertion = idx + break + + if idx_insertion is None: + pass + + lines.insert(idx_insertion + 1, UNDEF_STATEMENT) + with open(fname, 'w') as fh: + fh.write("".join(lines)) + + +def strip_leader(s, prefix): + """Remove given prefix from underscored name.""" + leader = prefix + "_" + if s.startswith(leader): + return s[len(leader) :] + else: + return s + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser() + # fmt: off + parser.add_argument("proto", nargs="+", help="Protobuf definition files") + parser.add_argument("-o", "--out-dir", help="Directory for generated source code") + parser.add_argument("-n", "--namespace", default=None, help="Message namespace") + parser.add_argument("-I", "--protoc-include", action="append", help="protoc include path") + parser.add_argument("-P", "--protobuf-module", default="protobuf", help="Name of protobuf module") + parser.add_argument("-f", "--force", default=False, help="Overwrite existing files") + # fmt: on + args = parser.parse_args() + + protoc_includes = args.protoc_include or (os.environ.get("PROTOC_INCLUDE"),) + + protoc( + args.proto, args.out_dir, protoc_includes, package=args.namespace, force=args.force + ) + + diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp new file mode 100644 index 000000000..fc86177e1 --- /dev/null +++ b/src/device_trezor/trezor/transport.cpp @@ -0,0 +1,651 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include <boost/endian/conversion.hpp> +#include <boost/asio/io_service.hpp> +#include <boost/asio/ip/udp.hpp> +#include <boost/date_time/posix_time/posix_time_types.hpp> +#include "transport.hpp" +#include "messages/messages-common.pb.h" + +using namespace std; +using json = rapidjson::Document; + + +namespace hw{ +namespace trezor{ + + bool t_serialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_serialize(const json_val & in, std::string & out){ + rapidjson::StringBuffer sb; + rapidjson::Writer<rapidjson::StringBuffer> writer(sb); + in.Accept(writer); + out = sb.GetString(); + return true; + } + + std::string t_serialize(const json_val & in){ + std::string ret; + t_serialize(in, ret); + return ret; + } + + bool t_deserialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_deserialize(const std::string & in, json & out){ + if (out.Parse(in.c_str()).HasParseError()) { + throw exc::CommunicationException("JSON parse error"); + } + return true; + } + + static std::string json_get_string(const rapidjson::Value & in){ + return std::string(in.GetString()); + } + + // + // Helpers + // + +#define PROTO_HEADER_SIZE 6 + + static size_t message_size(const google::protobuf::Message &req){ + return static_cast<size_t>(req.ByteSize()); + } + + static size_t serialize_message_buffer_size(size_t msg_size) { + return PROTO_HEADER_SIZE + msg_size; // tag 2B + len 4B + } + + static void serialize_message_header(void * buff, uint16_t tag, uint32_t len){ + uint16_t wire_tag = boost::endian::native_to_big(static_cast<uint16_t>(tag)); + uint32_t wire_len = boost::endian::native_to_big(static_cast<uint32_t>(len)); + memcpy(buff, (void *) &wire_tag, 2); + memcpy((uint8_t*)buff + 2, (void *) &wire_len, 4); + } + + static void deserialize_message_header(const void * buff, uint16_t & tag, uint32_t & len){ + uint16_t wire_tag; + uint32_t wire_len; + memcpy(&wire_tag, buff, 2); + memcpy(&wire_len, (uint8_t*)buff + 2, 4); + + tag = boost::endian::big_to_native(wire_tag); + len = boost::endian::big_to_native(wire_len); + } + + static void serialize_message(const google::protobuf::Message &req, size_t msg_size, uint8_t * buff, size_t buff_size) { + auto msg_wire_num = MessageMapper::get_message_wire_number(req); + const auto req_buffer_size = serialize_message_buffer_size(msg_size); + if (req_buffer_size > buff_size){ + throw std::invalid_argument("Buffer too small"); + } + + serialize_message_header(buff, msg_wire_num, msg_size); + if (!req.SerializeToArray(buff + 6, msg_size)){ + throw exc::EncodingException("Message serialization error"); + } + } + + // + // Communication protocol + // + +#define REPLEN 64 + + void ProtocolV1::write(Transport & transport, const google::protobuf::Message & req){ + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size) + 2; + + std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + req_buff_raw[0] = '#'; + req_buff_raw[1] = '#'; + + serialize_message(req, msg_size, req_buff_raw + 2, buff_size - 2); + + size_t offset = 0; + uint8_t chunk_buff[REPLEN]; + + // Chunk by chunk upload + while(offset < buff_size){ + auto to_copy = std::min((size_t)(buff_size - offset), (size_t)(REPLEN - 1)); + + chunk_buff[0] = '?'; + memcpy(chunk_buff + 1, req_buff_raw + offset, to_copy); + + // Pad with zeros + if (to_copy < REPLEN - 1){ + memset(chunk_buff + 1 + to_copy, 0, REPLEN - 1 - to_copy); + } + + transport.write_chunk(chunk_buff, REPLEN); + offset += REPLEN - 1; + } + } + + void ProtocolV1::read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type){ + char chunk[REPLEN]; + + // Initial chunk read + size_t nread = transport.read_chunk(chunk, REPLEN); + if (nread != REPLEN){ + throw exc::CommunicationException("Read chunk has invalid size"); + } + + if (strncmp(chunk, "?##", 3) != 0){ + throw exc::CommunicationException("Malformed chunk"); + } + + uint16_t tag; + uint32_t len; + nread -= 3 + 6; + deserialize_message_header(chunk + 3, tag, len); + + std::string data_acc(chunk + 3 + 6, nread); + data_acc.reserve(len); + + while(nread < len){ + const size_t cur = transport.read_chunk(chunk, REPLEN); + if (chunk[0] != '?'){ + throw exc::CommunicationException("Chunk malformed"); + } + + data_acc.append(chunk + 1, cur - 1); + nread += cur - 1; + } + + if (msg_type){ + *msg_type = static_cast<messages::MessageType>(tag); + } + + if (nread < len){ + throw exc::CommunicationException("Response incomplete"); + } + + std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(tag)); + if (!msg_wrap->ParseFromArray(data_acc.c_str(), len)){ + throw exc::CommunicationException("Message could not be parsed"); + } + + msg = msg_wrap; + } + + // + // Bridge transport + // + + const char * BridgeTransport::PATH_PREFIX = "bridge:"; + + std::string BridgeTransport::get_path() const { + if (!m_device_path){ + return ""; + } + + std::string path(PATH_PREFIX); + return path + m_device_path.get(); + } + + void BridgeTransport::enumerate(t_transport_vect & res) { + json bridge_res; + std::string req; + + bool req_status = invoke_bridge_http("/enumerate", req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Bridge enumeration failed"); + } + + for(rapidjson::Value::ConstValueIterator itr = bridge_res.Begin(); itr != bridge_res.End(); ++itr){ + auto element = itr->GetObject(); + auto t = std::make_shared<BridgeTransport>(boost::make_optional(json_get_string(element["path"]))); + t->m_device_info.emplace(); + t->m_device_info->CopyFrom(*itr, t->m_device_info->GetAllocator()); + res.push_back(t); + } + } + + void BridgeTransport::open() { + if (!m_device_path){ + throw exc::CommunicationException("Coud not open, empty device path"); + } + + std::string uri = "/acquire/" + m_device_path.get() + "/null"; + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to acquire device"); + } + + m_session = boost::make_optional(json_get_string(bridge_res["session"])); + } + + void BridgeTransport::close() { + if (!m_device_path || !m_session){ + throw exc::CommunicationException("Device not open"); + } + + std::string uri = "/release/" + m_session.get(); + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to release device"); + } + + m_session = boost::none; + } + + void BridgeTransport::write(const google::protobuf::Message &req) { + m_response = boost::none; + + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size); + + std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + + serialize_message(req, msg_size, req_buff_raw, buff_size); + + std::string uri = "/call/" + m_session.get(); + std::string req_hex = epee::to_hex::string(epee::span<const std::uint8_t>(req_buff_raw, buff_size)); + std::string res_hex; + + bool req_status = invoke_bridge_http(uri, req_hex, res_hex, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Call method failed"); + } + + m_response = res_hex; + } + + void BridgeTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + if (!m_response){ + throw exc::CommunicationException("Could not read, no response stored"); + } + + std::string bin_data; + if (!epee::string_tools::parse_hexstr_to_binbuff(m_response.get(), bin_data)){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + uint16_t msg_tag; + uint32_t msg_len; + deserialize_message_header(bin_data.c_str(), msg_tag, msg_len); + if (bin_data.size() != msg_len + 6){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + if (msg_type){ + *msg_type = static_cast<messages::MessageType>(msg_tag); + } + + std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag)); + if (!msg_wrap->ParseFromArray(bin_data.c_str() + 6, msg_len)){ + throw exc::EncodingException("Response is not well hexcoded"); + } + msg = msg_wrap; + } + + const boost::optional<json> & BridgeTransport::device_info() const { + return m_device_info; + } + + std::ostream& BridgeTransport::dump(std::ostream& o) const { + return o << "BridgeTransport<path=" << (m_device_path ? get_path() : "None") + << ", info=" << (m_device_info ? t_serialize(m_device_info.get()) : "None") + << ", session=" << (m_session ? m_session.get() : "None") + << ">"; + } + + // + // UdpTransport + // + const char * UdpTransport::PATH_PREFIX = "udp:"; + const char * UdpTransport::DEFAULT_HOST = "127.0.0.1"; + const int UdpTransport::DEFAULT_PORT = 21324; + + UdpTransport::UdpTransport(boost::optional<std::string> device_path, + boost::optional<std::shared_ptr<Protocol>> proto) : + m_io_service(), m_deadline(m_io_service) + { + m_device_port = DEFAULT_PORT; + if (device_path) { + const std::string device_str = device_path.get(); + auto delim = device_str.find(':'); + if (delim == std::string::npos) { + m_device_host = device_str; + } else { + m_device_host = device_str.substr(0, delim); + m_device_port = std::stoi(device_str.substr(delim + 1)); + } + } else { + m_device_host = DEFAULT_HOST; + } + + if (m_device_port <= 1024 || m_device_port > 65535){ + throw std::invalid_argument("Port number invalid"); + } + + if (m_device_host != "localhost" && m_device_host != DEFAULT_HOST){ + throw std::invalid_argument("Local endpoint allowed only"); + } + + m_proto = proto ? proto.get() : std::make_shared<ProtocolV1>(); + } + + std::string UdpTransport::get_path() const { + std::string path(PATH_PREFIX); + return path + m_device_host + ":" + std::to_string(m_device_port); + } + + void UdpTransport::require_socket(){ + if (!m_socket){ + throw exc::NotConnectedException("Socket not connected"); + } + } + + bool UdpTransport::ping(){ + return ping_int(); + } + + bool UdpTransport::ping_int(boost::posix_time::time_duration timeout){ + require_socket(); + try { + std::string req = "PINGPING"; + char res[8]; + + m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); + receive(res, 8, nullptr, false, timeout); + + return memcmp(res, "PONGPONG", 8) == 0; + + } catch(...){ + return false; + } + } + + void UdpTransport::enumerate(t_transport_vect & res) { + std::shared_ptr<UdpTransport> t = std::make_shared<UdpTransport>(); + bool t_works = false; + + try{ + t->open(); + t_works = t->ping(); + } catch(...) { + + } + t->close(); + if (t_works){ + res.push_back(t); + } + } + + void UdpTransport::open() { + udp::resolver resolver(m_io_service); + udp::resolver::query query(udp::v4(), m_device_host, std::to_string(m_device_port)); + m_endpoint = *resolver.resolve(query); + + m_socket.reset(new udp::socket(m_io_service)); + m_socket->open(udp::v4()); + + m_deadline.expires_at(boost::posix_time::pos_infin); + check_deadline(); + + m_proto->session_begin(*this); + } + + void UdpTransport::close() { + if (!m_socket){ + throw exc::CommunicationException("Socket is already closed"); + } + + m_proto->session_end(*this); + m_socket->close(); + m_socket = nullptr; + } + + void UdpTransport::write_chunk(const void * buff, size_t size){ + require_socket(); + + if (size != 64){ + throw exc::CommunicationException("Invalid chunk size"); + } + + auto written = m_socket->send_to(boost::asio::buffer(buff, size), m_endpoint); + if (size != written){ + throw exc::CommunicationException("Could not send the whole chunk"); + } + } + + size_t UdpTransport::read_chunk(void * buff, size_t size){ + require_socket(); + if (size < 64){ + throw std::invalid_argument("Buffer too small"); + } + + ssize_t len; + while(true) { + try { + boost::system::error_code ec; + len = receive(buff, size, &ec, true); + if (ec == boost::asio::error::operation_aborted) { + continue; + } else if (ec) { + throw exc::CommunicationException(std::string("Comm error: ") + ec.message()); + } + + if (len != 64) { + throw exc::CommunicationException("Invalid chunk size"); + } + + break; + + } catch(exc::CommunicationException const& e){ + throw; + } catch(std::exception const& e){ + MWARNING("Error reading chunk, reason: " << e.what()); + throw exc::CommunicationException(std::string("Chunk read error: ") + std::string(e.what())); + } + } + + return static_cast<size_t>(len); + } + + ssize_t UdpTransport::receive(void * buff, size_t size, boost::system::error_code * error_code, bool no_throw, boost::posix_time::time_duration timeout){ + boost::system::error_code ec; + boost::asio::mutable_buffer buffer = boost::asio::buffer(buff, size); + + require_socket(); + + // Set a deadline for the asynchronous operation. + m_deadline.expires_from_now(timeout); + + // Set up the variables that receive the result of the asynchronous + // operation. The error code is set to would_block to signal that the + // operation is incomplete. Asio guarantees that its asynchronous + // operations will never fail with would_block, so any other value in + // ec indicates completion. + ec = boost::asio::error::would_block; + std::size_t length = 0; + + // Start the asynchronous operation itself. The handle_receive function + // used as a callback will update the ec and length variables. + m_socket->async_receive_from(boost::asio::buffer(buffer), m_endpoint, + boost::bind(&UdpTransport::handle_receive, _1, _2, &ec, &length)); + + // Block until the asynchronous operation has completed. + do { + m_io_service.run_one(); + } + while (ec == boost::asio::error::would_block); + + if (error_code){ + *error_code = ec; + } + + if (no_throw){ + return length; + } + + // Operation result + if (ec == boost::asio::error::operation_aborted){ + throw exc::TimeoutException(); + + } else if (ec) { + MWARNING("Reading from UDP socket failed: " << ec.message()); + throw exc::CommunicationException(); + + } + + return length; + } + + void UdpTransport::write(const google::protobuf::Message &req) { + m_proto->write(*this, req); + } + + void UdpTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + m_proto->read(*this, msg, msg_type); + } + + void UdpTransport::check_deadline(){ + if (!m_socket){ + return; // no active socket. + } + + // Check whether the deadline has passed. We compare the deadline against + // the current time since a new asynchronous operation may have moved the + // deadline before this actor had a chance to run. + if (m_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) + { + // The deadline has passed. The outstanding asynchronous operation needs + // to be cancelled so that the blocked receive() function will return. + // + // Please note that cancel() has portability issues on some versions of + // Microsoft Windows, and it may be necessary to use close() instead. + // Consult the documentation for cancel() for further information. + m_socket->cancel(); + + // There is no longer an active deadline. The expiry is set to positive + // infinity so that the actor takes no action until a new deadline is set. + m_deadline.expires_at(boost::posix_time::pos_infin); + } + + // Put the actor back to sleep. + m_deadline.async_wait(boost::bind(&UdpTransport::check_deadline, this)); + } + + void UdpTransport::handle_receive(const boost::system::error_code &ec, std::size_t length, + boost::system::error_code *out_ec, std::size_t *out_length) { + *out_ec = ec; + *out_length = length; + } + + std::ostream& UdpTransport::dump(std::ostream& o) const { + return o << "UdpTransport<path=" << get_path() + << ", socket_alive=" << (m_socket ? "true" : "false") + << ">"; + } + + void enumerate(t_transport_vect & res){ + BridgeTransport bt; + bt.enumerate(res); + + hw::trezor::UdpTransport btu; + btu.enumerate(res); + } + + std::shared_ptr<Transport> transport(const std::string & path){ + if (boost::starts_with(path, BridgeTransport::PATH_PREFIX)){ + return std::make_shared<BridgeTransport>(path.substr(strlen(BridgeTransport::PATH_PREFIX))); + + } else if (boost::starts_with(path, UdpTransport::PATH_PREFIX)){ + return std::make_shared<UdpTransport>(path.substr(strlen(UdpTransport::PATH_PREFIX))); + + } else { + throw std::invalid_argument("Unknown Trezor device path: " + path); + + } + } + + void throw_failure_exception(const messages::common::Failure * failure) { + if (failure == nullptr){ + throw std::invalid_argument("Failure message cannot be null"); + } + + boost::optional<std::string> message = failure->has_message() ? boost::make_optional(failure->message()) : boost::none; + boost::optional<uint32_t> code = failure->has_code() ? boost::make_optional(static_cast<uint32_t>(failure->code())) : boost::none; + if (!code){ + throw exc::proto::FailureException(code, message); + } + + auto ecode = failure->code(); + if (ecode == messages::common::Failure_FailureType_Failure_UnexpectedMessage){ + throw exc::proto::UnexpectedMessageException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_ActionCancelled){ + throw exc::proto::CancelledException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinExpected){ + throw exc::proto::PinExpectedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinInvalid){ + throw exc::proto::InvalidPinException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotEnoughFunds){ + throw exc::proto::NotEnoughFundsException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotInitialized){ + throw exc::proto::NotInitializedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_FirmwareError){ + throw exc::proto::FirmwareErrorException(code, message); + } else { + throw exc::proto::FailureException(code, message); + } + } + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t){ + return t.dump(o); + } + + std::ostream& operator<<(std::ostream& o, std::shared_ptr<hw::trezor::Transport> const& t){ + if (!t){ + return o << "None"; + } + + return t->dump(o); + } + +} +} + + diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp new file mode 100644 index 000000000..7b82fd06f --- /dev/null +++ b/src/device_trezor/trezor/transport.hpp @@ -0,0 +1,331 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TRANSPORT_H +#define MONERO_TRANSPORT_H + + +#include <boost/asio.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/array.hpp> +#include <boost/utility/string_ref.hpp> + +#include <typeinfo> +#include <type_traits> +#include "net/http_client.h" + +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" + +#include "exceptions.hpp" +#include "trezor_defs.hpp" +#include "messages_map.hpp" + +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +namespace hw { +namespace trezor { + + using json = rapidjson::Document; + using json_val = rapidjson::Value; + namespace http = epee::net_utils::http; + + const std::string DEFAULT_BRIDGE = "127.0.0.1:21325"; + + // Base HTTP comm serialization. + bool t_serialize(const std::string & in, std::string & out); + bool t_serialize(const json_val & in, std::string & out); + std::string t_serialize(const json_val & in); + + bool t_deserialize(const std::string & in, std::string & out); + bool t_deserialize(const std::string & in, json & out); + + // Flexible json serialization. HTTP client tailored for bridge API + template<class t_req, class t_res, class t_transport> + bool invoke_bridge_http(const boost::string_ref uri, const t_req & out_struct, t_res & result_struct, t_transport& transport, const boost::string_ref method = "POST", std::chrono::milliseconds timeout = std::chrono::seconds(180)) + { + std::string req_param; + t_serialize(out_struct, req_param); + + http::fields_list additional_params; + additional_params.push_back(std::make_pair("Origin","https://monero.trezor.io")); + additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8")); + + const http::http_response_info* pri = nullptr; + if(!transport.invoke(uri, method, req_param, timeout, &pri, std::move(additional_params))) + { + MERROR("Failed to invoke http request to " << uri); + return false; + } + + if(!pri) + { + MERROR("Failed to invoke http request to " << uri << ", internal error (null response ptr)"); + return false; + } + + if(pri->m_response_code != 200) + { + MERROR("Failed to invoke http request to " << uri << ", wrong response code: " << pri->m_response_code + << " Response Body: " << pri->m_body); + return false; + } + + return t_deserialize(pri->m_body, result_struct); + } + + // Forward decl + class Transport; + class Protocol; + + // Communication protocol + class Protocol { + public: + Protocol() = default; + virtual ~Protocol() = default; + virtual void session_begin(Transport & transport){ }; + virtual void session_end(Transport & transport){ }; + virtual void write(Transport & transport, const google::protobuf::Message & req)= 0; + virtual void read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr)= 0; + }; + + class ProtocolV1 : public Protocol { + public: + ProtocolV1() = default; + virtual ~ProtocolV1() = default; + + void write(Transport & transport, const google::protobuf::Message & req) override; + void read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + }; + + + // Base transport + typedef std::vector<std::shared_ptr<Transport>> t_transport_vect; + + class Transport { + public: + Transport() = default; + virtual ~Transport() = default; + + virtual bool ping() { return false; }; + virtual std::string get_path() const { return ""; }; + virtual void enumerate(t_transport_vect & res){}; + virtual void open(){}; + virtual void close(){}; + virtual void write(const google::protobuf::Message & req) =0; + virtual void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) =0; + + virtual void write_chunk(const void * buff, size_t size) { }; + virtual size_t read_chunk(void * buff, size_t size) { return 0; }; + virtual std::ostream& dump(std::ostream& o) const { return o << "Transport<>"; } + }; + + // Bridge transport + class BridgeTransport : public Transport { + public: + BridgeTransport( + boost::optional<std::string> device_path = boost::none, + boost::optional<std::string> bridge_host = boost::none): + m_device_path(device_path), + m_bridge_host(bridge_host ? bridge_host.get() : DEFAULT_BRIDGE), + m_response(boost::none), + m_session(boost::none), + m_device_info(boost::none) + { + m_http_client.set_server(m_bridge_host, boost::none, false); + } + + virtual ~BridgeTransport() = default; + + static const char * PATH_PREFIX; + + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + const boost::optional<json> & device_info() const; + std::ostream& dump(std::ostream& o) const override; + + private: + epee::net_utils::http::http_simple_client m_http_client; + std::string m_bridge_host; + boost::optional<std::string> m_device_path; + boost::optional<std::string> m_session; + boost::optional<std::string> m_response; + boost::optional<json> m_device_info; + }; + + // UdpTransport transport + using boost::asio::ip::udp; + + class UdpTransport : public Transport { + public: + + explicit UdpTransport( + boost::optional<std::string> device_path=boost::none, + boost::optional<std::shared_ptr<Protocol>> proto=boost::none); + + virtual ~UdpTransport() = default; + + static const char * PATH_PREFIX; + static const char * DEFAULT_HOST; + static const int DEFAULT_PORT; + + bool ping() override; + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + void write_chunk(const void * buff, size_t size) override; + size_t read_chunk(void * buff, size_t size) override; + + std::ostream& dump(std::ostream& o) const override; + + private: + void require_socket(); + ssize_t receive(void * buff, size_t size, boost::system::error_code * error_code=nullptr, bool no_throw=false, boost::posix_time::time_duration timeout=boost::posix_time::seconds(10)); + void check_deadline(); + static void handle_receive(const boost::system::error_code& ec, std::size_t length, + boost::system::error_code* out_ec, std::size_t* out_length); + bool ping_int(boost::posix_time::time_duration timeout=boost::posix_time::milliseconds(1500)); + + std::shared_ptr<Protocol> m_proto; + std::string m_device_host; + int m_device_port; + + std::unique_ptr<udp::socket> m_socket; + boost::asio::io_service m_io_service; + boost::asio::deadline_timer m_deadline; + udp::endpoint m_endpoint; + }; + + // + // General helpers + // + + /** + * Enumerates all transports + */ + void enumerate(t_transport_vect & res); + + /** + * Transforms path to the transport + */ + std::shared_ptr<Transport> transport(const std::string & path); + + /** + * Transforms path to the particular transport + */ + template<class t_transport> + std::shared_ptr<t_transport> transport_typed(const std::string & path){ + auto t = transport(path); + if (!t){ + return nullptr; + } + + return std::dynamic_pointer_cast<t_transport>(t); + } + + // Exception carries unexpected message being received + namespace exc { + class UnexpectedMessageException: public ProtocolException { + protected: + hw::trezor::messages::MessageType recvType; + std::shared_ptr<google::protobuf::Message> recvMsg; + + public: + using ProtocolException::ProtocolException; + UnexpectedMessageException(): ProtocolException("Trezor returned unexpected message") {}; + UnexpectedMessageException(hw::trezor::messages::MessageType recvType, + const std::shared_ptr<google::protobuf::Message> & recvMsg) + : recvType(recvType), recvMsg(recvMsg) { + reason = std::string("Trezor returned unexpected message: ") + std::to_string(recvType); + } + }; + } + + /** + * Throws corresponding failure exception. + */ + [[ noreturn ]] void throw_failure_exception(const messages::common::Failure * failure); + + /** + * Simple wrapper for write-read message exchange with expected message response type. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template<class t_message> + std::shared_ptr<t_message> + exchange_message(Transport & transport, const google::protobuf::Message & req, + boost::optional<messages::MessageType> resp_type = boost::none) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + + // Write the request + transport.write(req); + + // Read the response + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + transport.read(msg_resp, &msg_resp_type); + + // Determine type of expected message response + messages::MessageType required_type = resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>(); + + if (msg_resp_type == required_type) { + return message_ptr_retype<t_message>(msg_resp); + } else if (msg_resp_type == messages::MessageType_Failure){ + throw_failure_exception(dynamic_cast<messages::common::Failure*>(msg_resp.get())); + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + } + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t); + std::ostream& operator<<(std::ostream& o, std::shared_ptr<hw::trezor::Transport> const& t); +}} + + +#endif //MONERO_TRANSPORT_H diff --git a/src/device_trezor/trezor/trezor_defs.hpp b/src/device_trezor/trezor/trezor_defs.hpp new file mode 100644 index 000000000..951a8f802 --- /dev/null +++ b/src/device_trezor/trezor/trezor_defs.hpp @@ -0,0 +1,48 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#if defined(HAVE_PROTOBUF) && !defined(WITHOUT_TREZOR) + #define WITH_DEVICE_TREZOR 1 +#else + #define WITH_DEVICE_TREZOR 0 +#endif + +#ifndef WITH_DEVICE_TREZOR_LITE +#define WITH_DEVICE_TREZOR_LITE 0 +#endif + +// Avoids protobuf undefined macro warning +#ifndef PROTOBUF_INLINE_NOT_IN_HEADERS +#define PROTOBUF_INLINE_NOT_IN_HEADERS 0 +#endif + +// Fixes gcc7 problem with minor macro defined clashing with minor() field. +#ifdef minor +#undef minor +#endif diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index 03e0a7946..8262e86f7 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -91,7 +91,7 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str for (size_t n = 0; n < total; ++n) { std::string name = basename + "-" + std::to_string(n + 1); - wallets[n].reset(new tools::wallet2(nettype)); + wallets[n].reset(new tools::wallet2(nettype, 1, false)); wallets[n]->init(""); wallets[n]->generate(name, pwd_container->password(), rct::rct2sk(rct::skGen()), false, false, create_address_file); } @@ -101,11 +101,13 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str std::vector<crypto::public_key> pk(total); for (size_t n = 0; n < total; ++n) { + wallets[n]->decrypt_keys(pwd_container->password()); if (!tools::wallet2::verify_multisig_info(wallets[n]->get_multisig_info(), sk[n], pk[n])) { - tools::fail_msg_writer() << tr("Failed to verify multisig info"); + tools::fail_msg_writer() << genms::tr("Failed to verify multisig info"); return false; } + wallets[n]->encrypt_keys(pwd_container->password()); } // make the wallets multisig @@ -128,8 +130,8 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str ss << " " << name << std::endl; } - // finalize step if needed - if (!extra_info[0].empty()) + //exchange keys unless exchange_multisig_keys returns no extra info + while (!extra_info[0].empty()) { std::unordered_set<crypto::public_key> pkeys; std::vector<crypto::public_key> signers(total); @@ -143,11 +145,7 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str } for (size_t n = 0; n < total; ++n) { - if (!wallets[n]->finalize_multisig(pwd_container->password(), pkeys, signers)) - { - tools::fail_msg_writer() << genms::tr("Error finalizing multisig"); - return false; - } + extra_info[n] = wallets[n]->exchange_multisig_keys(pwd_container->password(), pkeys, signers); } } @@ -165,6 +163,8 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str int main(int argc, char* argv[]) { + TRY_ENTRY(); + po::options_description desc_params(wallet_args::tr("Wallet options")); command_line::add_arg(desc_params, arg_filename_base); command_line::add_arg(desc_params, arg_scheme); @@ -242,15 +242,10 @@ int main(int argc, char* argv[]) return 1; } - if (threshold != total-1 && threshold != total) - { - tools::fail_msg_writer() << genms::tr("Error: unsupported scheme: only N/N and N-1/N are supported"); - return 1; - } bool create_address_file = command_line::get_arg(*vm, arg_create_address_file); if (!generate_multisig(threshold, total, basename, testnet ? TESTNET : stagenet ? STAGENET : MAINNET, create_address_file)) return 1; return 0; - //CATCH_ENTRY_L0("main", 1); + CATCH_ENTRY_L0("main", 1); } diff --git a/src/mnemonics/chinese_simplified.h b/src/mnemonics/chinese_simplified.h index 1ae8c89d6..0566b1079 100644 --- a/src/mnemonics/chinese_simplified.h +++ b/src/mnemonics/chinese_simplified.h @@ -72,7 +72,10 @@ namespace Language class Chinese_Simplified: public Base
{
public:
- Chinese_Simplified(): Base("简体中文 (中国)", "Chinese (simplified)", std::vector<std::string>({
+ Chinese_Simplified(): Base("简体中文 (中国)", "Chinese (simplified)", {}, 1)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"的",
"一",
"是",
@@ -1699,8 +1702,8 @@ namespace Language "秒",
"浙",
"貌"
- }), 1)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/dutch.h b/src/mnemonics/dutch.h index c9806f450..801caf986 100644 --- a/src/mnemonics/dutch.h +++ b/src/mnemonics/dutch.h @@ -49,7 +49,10 @@ namespace Language class Dutch: public Base
{
public:
- Dutch(): Base("Nederlands", "Dutch", std::vector<std::string>({
+ Dutch(): Base("Nederlands", "Dutch", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"aalglad",
"aalscholver",
"aambeeld",
@@ -1676,8 +1679,8 @@ namespace Language "zwiep",
"zwijmel",
"zworen"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 6a2a3e0c4..496e26858 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -43,8 +43,11 @@ #include <vector> #include <unordered_map> #include <boost/algorithm/string.hpp> +#include "wipeable_string.h" +#include "misc_language.h" #include "crypto/crypto.h" // for declaration of crypto::secret_key #include <fstream> +#include "common/int-util.h" #include "mnemonics/electrum-words.h" #include <stdexcept> #include <boost/filesystem.hpp> @@ -67,11 +70,22 @@ #include "language_base.h" #include "singleton.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "mnemonic" + +namespace crypto +{ + namespace ElectrumWords + { + std::vector<const Language::Base*> get_language_list(); + } +} + namespace { - uint32_t create_checksum_index(const std::vector<std::string> &word_list, + uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, uint32_t unique_prefix_length); - bool checksum_test(std::vector<std::string> seed, uint32_t unique_prefix_length); + bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length); /*! * \brief Finds the word list that contains the seed words and puts the indices @@ -82,7 +96,7 @@ namespace * \param language Language instance pointer to write to after it is found. * \return true if all the words were present in some language false if not. */ - bool find_seed_language(const std::vector<std::string> &seed, + bool find_seed_language(const std::vector<epee::wipeable_string> &seed, bool has_checksum, std::vector<uint32_t> &matched_indices, Language::Base **language) { // If there's a new language added, add an instance of it here. @@ -103,17 +117,19 @@ namespace }); Language::Base *fallback = NULL; + std::vector<epee::wipeable_string>::const_iterator it2; + matched_indices.reserve(seed.size()); + // Iterate through all the languages and find a match for (std::vector<Language::Base*>::iterator it1 = language_instances.begin(); it1 != language_instances.end(); it1++) { - const std::unordered_map<std::string, uint32_t> &word_map = (*it1)->get_word_map(); - const std::unordered_map<std::string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t> &word_map = (*it1)->get_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); // To iterate through seed words - std::vector<std::string>::const_iterator it2; bool full_match = true; - std::string trimmed_word; + epee::wipeable_string trimmed_word; // Iterate through all the words and see if they're all present for (it2 = seed.begin(); it2 != seed.end(); it2++) { @@ -152,9 +168,11 @@ namespace if (full_match) { *language = *it1; + MINFO("Full match for language " << (*language)->get_english_language_name()); return true; } // Some didn't match. Clear the index array. + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); matched_indices.clear(); } @@ -164,9 +182,12 @@ namespace if (fallback) { *language = fallback; + MINFO("Fallback match for language " << (*language)->get_english_language_name()); return true; } + MINFO("No match found"); + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); return false; } @@ -176,12 +197,12 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return Checksum index */ - uint32_t create_checksum_index(const std::vector<std::string> &word_list, + uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, uint32_t unique_prefix_length) { - std::string trimmed_words = ""; + epee::wipeable_string trimmed_words = ""; - for (std::vector<std::string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) + for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) { if (it->length() > unique_prefix_length) { @@ -194,7 +215,7 @@ namespace } boost::crc_32_type result; result.process_bytes(trimmed_words.data(), trimmed_words.length()); - return result.checksum() % crypto::ElectrumWords::seed_length; + return result.checksum() % word_list.size(); } /*! @@ -203,21 +224,23 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return True if the test passed false if not. */ - bool checksum_test(std::vector<std::string> seed, uint32_t unique_prefix_length) + bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length) { if (seed.empty()) return false; // The last word is the checksum. - std::string last_word = seed.back(); + epee::wipeable_string last_word = seed.back(); seed.pop_back(); - std::string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; + epee::wipeable_string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; - std::string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : + epee::wipeable_string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : checksum; - std::string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : + epee::wipeable_string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; - return trimmed_checksum == trimmed_last_word; + bool ret = trimmed_checksum == trimmed_last_word; + MINFO("Checksum is " << (ret ? "valid" : "invalid")); + return ret; } } @@ -244,16 +267,18 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name) { - std::vector<std::string> seed; + std::vector<epee::wipeable_string> seed; - boost::algorithm::trim(words); - boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); + words.split(seed); if (len % 4) + { + MERROR("Invalid seed: not a multiple of 4"); return false; + } bool has_checksum = true; if (len) @@ -263,6 +288,7 @@ namespace crypto if (seed.size() != expected/2 && seed.size() != expected && seed.size() != expected + 1) { + MERROR("Invalid seed: unexpected number of words"); return false; } @@ -271,9 +297,11 @@ namespace crypto } std::vector<uint32_t> matched_indices; + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0]));}); Language::Base *language; if (!find_seed_language(seed, has_checksum, matched_indices, &language)) { + MERROR("Invalid seed: language not found"); return false; } language_name = language->get_language_name(); @@ -284,6 +312,7 @@ namespace crypto if (!checksum_test(seed, language->get_unique_prefix_length())) { // Checksum fail + MERROR("Invalid seed: invalid checksum"); return false; } seed.pop_back(); @@ -291,29 +320,33 @@ namespace crypto for (unsigned int i=0; i < seed.size() / 3; i++) { - uint32_t val; - uint32_t w1, w2, w3; - w1 = matched_indices[i*3]; - w2 = matched_indices[i*3 + 1]; - w3 = matched_indices[i*3 + 2]; + uint32_t w[4]; + w[1] = matched_indices[i*3]; + w[2] = matched_indices[i*3 + 1]; + w[3] = matched_indices[i*3 + 2]; - val = w1 + word_list_length * (((word_list_length - w1) + w2) % word_list_length) + - word_list_length * word_list_length * (((word_list_length - w2) + w3) % word_list_length); + w[0]= w[1] + word_list_length * (((word_list_length - w[1]) + w[2]) % word_list_length) + + word_list_length * word_list_length * (((word_list_length - w[2]) + w[3]) % word_list_length); - if (!(val % word_list_length == w1)) return false; + if (!(w[0]% word_list_length == w[1])) + { + memwipe(w, sizeof(w)); + MERROR("Invalid seed: mumble mumble"); + return false; + } - dst.append((const char*)&val, 4); // copy 4 bytes to position + dst.append((const char*)&w[0], 4); // copy 4 bytes to position + memwipe(w, sizeof(w)); } if (len > 0 && duplicate) { const size_t expected = len * 3 / 32; - std::string wlist_copy = words; if (seed.size() == expected/2) { - dst.append(dst); // if electrum 12-word seed, duplicate - wlist_copy += ' '; - wlist_copy += words; + dst += ' '; // if electrum 12-word seed, duplicate + dst += dst; // if electrum 12-word seed, duplicate + dst.pop_back(); // trailing space } } @@ -327,14 +360,20 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name) { - std::string s; + epee::wipeable_string s; if (!words_to_bytes(words, s, sizeof(dst), true, language_name)) + { + MERROR("Invalid seed: failed to convert words to bytes"); return false; + } if (s.size() != sizeof(dst)) + { + MERROR("Invalid seed: wrong output size"); return false; + } dst = *(const crypto::secret_key*)s.data(); return true; } @@ -346,100 +385,57 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name) { if (len % 4 != 0 || len == 0) return false; - Language::Base *language; - if (language_name == "English") + const Language::Base *language = NULL; + const std::vector<const Language::Base*> language_list = crypto::ElectrumWords::get_language_list(); + for (const Language::Base *l: language_list) { - language = Language::Singleton<Language::English>::instance(); + if (language_name == l->get_language_name() || language_name == l->get_english_language_name()) + language = l; } - else if (language_name == "Nederlands") - { - language = Language::Singleton<Language::Dutch>::instance(); - } - else if (language_name == "Français") - { - language = Language::Singleton<Language::French>::instance(); - } - else if (language_name == "Español") - { - language = Language::Singleton<Language::Spanish>::instance(); - } - else if (language_name == "Português") - { - language = Language::Singleton<Language::Portuguese>::instance(); - } - else if (language_name == "日本語") - { - language = Language::Singleton<Language::Japanese>::instance(); - } - else if (language_name == "Italiano") - { - language = Language::Singleton<Language::Italian>::instance(); - } - else if (language_name == "Deutsch") - { - language = Language::Singleton<Language::German>::instance(); - } - else if (language_name == "русский язык") - { - language = Language::Singleton<Language::Russian>::instance(); - } - else if (language_name == "简体中文 (中国)") - { - language = Language::Singleton<Language::Chinese_Simplified>::instance(); - } - else if (language_name == "Esperanto") - { - language = Language::Singleton<Language::Esperanto>::instance(); - } - else if (language_name == "Lojban") - { - language = Language::Singleton<Language::Lojban>::instance(); - } - else + if (!language) { return false; } const std::vector<std::string> &word_list = language->get_word_list(); // To store the words for random access to add the checksum word later. - std::vector<std::string> words_store; + std::vector<epee::wipeable_string> words_store; uint32_t word_list_length = word_list.size(); // 4 bytes -> 3 words. 8 digits base 16 -> 3 digits base 1626 - for (unsigned int i=0; i < len/4; i++, words += ' ') + for (unsigned int i=0; i < len/4; i++, words.push_back(' ')) { - uint32_t w1, w2, w3; - - uint32_t val; + uint32_t w[4]; - memcpy(&val, src + (i * 4), 4); + w[0] = SWAP32LE(*(const uint32_t*)(src + (i * 4))); - w1 = val % word_list_length; - w2 = ((val / word_list_length) + w1) % word_list_length; - w3 = (((val / word_list_length) / word_list_length) + w2) % word_list_length; + w[1] = w[0] % word_list_length; + w[2] = ((w[0] / word_list_length) + w[1]) % word_list_length; + w[3] = (((w[0] / word_list_length) / word_list_length) + w[2]) % word_list_length; - words += word_list[w1]; + words += word_list[w[1]]; words += ' '; - words += word_list[w2]; + words += word_list[w[2]]; words += ' '; - words += word_list[w3]; + words += word_list[w[3]]; + + words_store.push_back(word_list[w[1]]); + words_store.push_back(word_list[w[2]]); + words_store.push_back(word_list[w[3]]); - words_store.push_back(word_list[w1]); - words_store.push_back(word_list[w2]); - words_store.push_back(word_list[w3]); + memwipe(w, sizeof(w)); } - words.pop_back(); - words += (' ' + words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]); + words += words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]; return true; } - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name) { return bytes_to_words(src.data, sizeof(src), words, language_name); @@ -483,11 +479,10 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed) + bool get_is_old_style_seed(const epee::wipeable_string &seed) { - std::vector<std::string> word_list; - boost::algorithm::trim(seed); - boost::split(word_list, seed, boost::is_any_of(" "), boost::token_compress_on); + std::vector<epee::wipeable_string> word_list; + seed.split(word_list); return word_list.size() != (seed_length + 1); } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 856edb92a..5401b9779 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -44,6 +44,8 @@ #include <map> #include "crypto/crypto.h" // for declaration of crypto::secret_key +namespace epee { class wipeable_string; } + /*! * \namespace crypto * @@ -70,7 +72,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name); /*! * \brief Converts seed words to bytes (secret key). @@ -79,7 +81,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name); /*! @@ -90,7 +92,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name); /*! @@ -100,7 +102,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name); /*! @@ -115,7 +117,7 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed); + bool get_is_old_style_seed(const epee::wipeable_string &seed); /*! * \brief Returns the name of a language in English diff --git a/src/mnemonics/english.h b/src/mnemonics/english.h index ee087674d..d5c5594ef 100644 --- a/src/mnemonics/english.h +++ b/src/mnemonics/english.h @@ -49,7 +49,10 @@ namespace Language class English: public Base
{
public:
- English(): Base("English", "English", std::vector<std::string>({
+ English(): Base("English", "English", {}, 3)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abbey",
"abducts",
"ability",
@@ -1676,8 +1679,8 @@ namespace Language "zombie",
"zones",
"zoom"
- }), 3)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/english_old.h b/src/mnemonics/english_old.h index b31491646..e35b907df 100644 --- a/src/mnemonics/english_old.h +++ b/src/mnemonics/english_old.h @@ -51,7 +51,10 @@ namespace Language class EnglishOld: public Base
{
public:
- EnglishOld(): Base("EnglishOld", "English (old)", std::vector<std::string>({
+ EnglishOld(): Base("EnglishOld", "English (old)", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"like",
"just",
"love",
@@ -1678,8 +1681,8 @@ namespace Language "unseen",
"weapon",
"weary"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps(ALLOW_DUPLICATE_PREFIXES | ALLOW_SHORT_WORDS);
}
};
diff --git a/src/mnemonics/esperanto.h b/src/mnemonics/esperanto.h index cb377a58e..b0be235ed 100644 --- a/src/mnemonics/esperanto.h +++ b/src/mnemonics/esperanto.h @@ -37,7 +37,7 @@ * Sources:
* Baza Radikaro Oficiala
* Reta Vortaro (http://www.reta-vortaro.de/revo/)
- * Esperanto Panorama - Esperanto-English Dictionary (http://www.esperanto-panorama.net/vortaro/eoen.htm)
+ * Esperanto Panorama - Esperanto-English Dictionary (https://www.esperanto-panorama.net/vortaro/eoen.htm)
* ESPDIC - Paul Denisowski (http://www.denisowski.org/Esperanto/ESPDIC/espdic.txt)
*/
@@ -58,7 +58,10 @@ namespace Language class Esperanto: public Base
{
public:
- Esperanto(): Base("Esperanto", "Esperanto", std::vector<std::string>({
+ Esperanto(): Base("Esperanto", "Esperanto", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abako",
"abdiki",
"abelo",
@@ -1685,8 +1688,8 @@ namespace Language "zorgi",
"zukino",
"zumilo",
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/french.h b/src/mnemonics/french.h index 7eaf45650..48ec46f78 100644 --- a/src/mnemonics/french.h +++ b/src/mnemonics/french.h @@ -49,7 +49,10 @@ namespace Language class French: public Base { public: - French(): Base("Français", "French", std::vector<std::string>({ + French(): Base("Français", "French", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "abandon", "abattre", "aboi", @@ -1676,8 +1679,8 @@ namespace Language "zinc", "zone", "zoom" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/german.h b/src/mnemonics/german.h index 8eff43523..883a173a3 100644 --- a/src/mnemonics/german.h +++ b/src/mnemonics/german.h @@ -51,7 +51,10 @@ namespace Language class German: public Base
{
public:
- German(): Base("Deutsch", "German", std::vector<std::string>({
+ German(): Base("Deutsch", "German", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"Abakus",
"Abart",
"abbilden",
@@ -1678,8 +1681,8 @@ namespace Language "Zündung",
"Zweck",
"Zyklop"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/italian.h b/src/mnemonics/italian.h index d5ecb74f4..57cdfa25e 100644 --- a/src/mnemonics/italian.h +++ b/src/mnemonics/italian.h @@ -51,7 +51,10 @@ namespace Language class Italian: public Base
{
public:
- Italian(): Base("Italiano", "Italian", std::vector<std::string>({
+ Italian(): Base("Italiano", "Italian", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abbinare",
"abbonato",
"abisso",
@@ -1678,8 +1681,8 @@ namespace Language "zolfo",
"zombie",
"zucchero"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/japanese.h b/src/mnemonics/japanese.h index f3b3e4924..5baabedf2 100644 --- a/src/mnemonics/japanese.h +++ b/src/mnemonics/japanese.h @@ -71,7 +71,10 @@ namespace Language class Japanese: public Base
{
public:
- Japanese(): Base("日本語", "Japanese", std::vector<std::string>({
+ Japanese(): Base("日本語", "Japanese", {}, 3)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"あいこくしん",
"あいさつ",
"あいだ",
@@ -1698,8 +1701,8 @@ namespace Language "ひさん",
"びじゅつかん",
"ひしょ"
- }), 3)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 2b0c37c6b..52e784cef 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -53,15 +53,20 @@ namespace Language * \param count How many characters to return.
* \return A string consisting of the first count characters in s.
*/
- inline std::string utf8prefix(const std::string &s, size_t count)
+ template<typename T>
+ inline T utf8prefix(const T &s, size_t count)
{
- std::string prefix = "";
- const char *ptr = s.c_str();
- while (count-- && *ptr)
+ T prefix = "";
+ size_t avail = s.size();
+ const char *ptr = s.data();
+ while (count-- && avail--)
{
prefix += *ptr++;
- while (((*ptr) & 0xc0) == 0x80)
+ while (avail && ((*ptr) & 0xc0) == 0x80)
+ {
prefix += *ptr++;
+ --avail;
+ }
}
return prefix;
}
@@ -78,9 +83,12 @@ namespace Language ALLOW_SHORT_WORDS = 1<<0,
ALLOW_DUPLICATE_PREFIXES = 1<<1,
};
- const std::vector<std::string> word_list; /*!< A pointer to the array of words */
- std::unordered_map<std::string, uint32_t> word_map; /*!< hash table to find word's index */
- std::unordered_map<std::string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
+ enum {
+ NWORDS = 1626
+ };
+ std::vector<std::string> word_list; /*!< A pointer to the array of words */
+ std::unordered_map<epee::wipeable_string, uint32_t> word_map; /*!< hash table to find word's index */
+ std::unordered_map<epee::wipeable_string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
std::string language_name; /*!< Name of language */
std::string english_language_name; /*!< Name of language */
uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
@@ -91,7 +99,7 @@ namespace Language {
int ii;
std::vector<std::string>::const_iterator it;
- if (word_list.size () != 1626)
+ if (word_list.size () != NWORDS)
throw std::runtime_error("Wrong word list length for " + language_name);
for (it = word_list.begin(), ii = 0; it != word_list.end(); it++, ii++)
{
@@ -103,7 +111,7 @@ namespace Language else
throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
}
- std::string trimmed;
+ epee::wipeable_string trimmed;
if (it->length() > unique_prefix_length)
{
trimmed = utf8prefix(*it, unique_prefix_length);
@@ -115,9 +123,9 @@ namespace Language if (trimmed_word_map.find(trimmed) != trimmed_word_map.end())
{
if (flags & ALLOW_DUPLICATE_PREFIXES)
- MWARNING("Duplicate prefix in " << language_name << " word list: " << trimmed);
+ MWARNING("Duplicate prefix in " << language_name << " word list: " << std::string(trimmed.data(), trimmed.size()));
else
- throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + trimmed);
+ throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + std::string(trimmed.data(), trimmed.size()));
}
trimmed_word_map[trimmed] = ii;
}
@@ -133,6 +141,12 @@ namespace Language virtual ~Base()
{
}
+ void set_words(const char * const words[])
+ {
+ word_list.resize(NWORDS);
+ for (size_t i = 0; i < NWORDS; ++i)
+ word_list[i] = words[i];
+ }
/*!
* \brief Returns a pointer to the word list.
* \return A pointer to the word list.
@@ -145,7 +159,7 @@ namespace Language * \brief Returns a pointer to the word map.
* \return A pointer to the word map.
*/
- const std::unordered_map<std::string, uint32_t>& get_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t>& get_word_map() const
{
return word_map;
}
@@ -153,7 +167,7 @@ namespace Language * \brief Returns a pointer to the trimmed word map.
* \return A pointer to the trimmed word map.
*/
- const std::unordered_map<std::string, uint32_t>& get_trimmed_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t>& get_trimmed_word_map() const
{
return trimmed_word_map;
}
diff --git a/src/mnemonics/lojban.h b/src/mnemonics/lojban.h index 412531a23..5162a8ec9 100644 --- a/src/mnemonics/lojban.h +++ b/src/mnemonics/lojban.h @@ -35,7 +35,7 @@ /*
* Word list authored by: sorpaas
* Sources:
- * lo gimste jo'u lo ma'oste (http://guskant.github.io/lojbo/gismu-cmavo.html)
+ * lo gimste jo'u lo ma'oste (https://guskant.github.io/lojbo/gismu-cmavo.html)
* N-grams of Lojban corpus (https://mw.lojban.org/papri/N-grams_of_Lojban_corpus)
*/
@@ -56,7 +56,10 @@ namespace Language class Lojban: public Base
{
public:
- Lojban(): Base("Lojban", "Lojban", std::vector<std::string>({
+ Lojban(): Base("Lojban", "Lojban", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"backi",
"bacru",
"badna",
@@ -1683,8 +1686,8 @@ namespace Language "noltruti'u",
"samtci",
"snaxa'a",
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/portuguese.h b/src/mnemonics/portuguese.h index 0195389ce..af04f89c2 100644 --- a/src/mnemonics/portuguese.h +++ b/src/mnemonics/portuguese.h @@ -72,7 +72,10 @@ namespace Language class Portuguese: public Base
{
public:
- Portuguese(): Base("Português", "Portuguese", std::vector<std::string>({
+ Portuguese(): Base("Português", "Portuguese", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abaular",
"abdominal",
"abeto",
@@ -1699,8 +1702,8 @@ namespace Language "zeloso",
"zenite",
"zumbi"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/russian.h b/src/mnemonics/russian.h index d5dd556ef..f3e70ede6 100644 --- a/src/mnemonics/russian.h +++ b/src/mnemonics/russian.h @@ -51,7 +51,10 @@ namespace Language class Russian: public Base
{
public:
- Russian(): Base("русский язык", "Russian", std::vector<std::string>({
+ Russian(): Base("русский язык", "Russian", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"абажур",
"абзац",
"абонент",
@@ -1678,8 +1681,8 @@ namespace Language "яхта",
"ячейка",
"ящик"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/spanish.h b/src/mnemonics/spanish.h index 51f38fede..4d7a896a6 100644 --- a/src/mnemonics/spanish.h +++ b/src/mnemonics/spanish.h @@ -72,7 +72,10 @@ namespace Language class Spanish: public Base
{
public:
- Spanish(): Base("Español", "Spanish", std::vector<std::string>({
+ Spanish(): Base("Español", "Spanish", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"ábaco",
"abdomen",
"abeja",
@@ -1699,8 +1702,8 @@ namespace Language "risa",
"ritmo",
"rito"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps(ALLOW_SHORT_WORDS);
}
};
diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index d85c47772..33d0a312f 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -47,9 +47,12 @@ namespace cryptonote crypto::secret_key get_multisig_blinded_secret_key(const crypto::secret_key &key) { rct::keyV data; + data.reserve(2); data.push_back(rct::sk2rct(key)); data.push_back(multisig_salt); - return rct::rct2sk(rct::hash_to_scalar(data)); + crypto::secret_key result = rct::rct2sk(rct::hash_to_scalar(data)); + memwipe(&data[0], sizeof(rct::key)); + return result; } //----------------------------------------------------------------- void generate_multisig_N_N(const account_keys &keys, const std::vector<crypto::public_key> &spend_keys, std::vector<crypto::secret_key> &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey) @@ -81,6 +84,43 @@ namespace cryptonote } } //----------------------------------------------------------------- + std::vector<crypto::public_key> generate_multisig_derivations(const account_keys &keys, const std::vector<crypto::public_key> &derivations) + { + std::vector<crypto::public_key> multisig_keys; + crypto::secret_key blinded_skey = get_multisig_blinded_secret_key(keys.m_spend_secret_key); + for (const auto &k: derivations) + { + rct::key d = rct::scalarmultKey(rct::pk2rct(k), rct::sk2rct(blinded_skey)); + multisig_keys.push_back(rct::rct2pk(d)); + } + + return multisig_keys; + } + //----------------------------------------------------------------- + crypto::secret_key calculate_multisig_signer_key(const std::vector<crypto::secret_key>& multisig_keys) + { + rct::key secret_key = rct::zero(); + for (const auto &k: multisig_keys) + { + sc_add(secret_key.bytes, secret_key.bytes, (const unsigned char*)k.data); + } + + return rct::rct2sk(secret_key); + } + //----------------------------------------------------------------- + std::vector<crypto::secret_key> calculate_multisig_keys(const std::vector<crypto::public_key>& derivations) + { + std::vector<crypto::secret_key> multisig_keys; + multisig_keys.reserve(derivations.size()); + + for (const auto &k: derivations) + { + multisig_keys.emplace_back(get_multisig_blinded_secret_key(rct::rct2sk(rct::pk2rct(k)))); + } + + return multisig_keys; + } + //----------------------------------------------------------------- crypto::secret_key generate_multisig_view_secret_key(const crypto::secret_key &skey, const std::vector<crypto::secret_key> &skeys) { rct::key view_skey = rct::sk2rct(get_multisig_blinded_secret_key(skey)); @@ -89,7 +129,7 @@ namespace cryptonote return rct::rct2sk(view_skey); } //----------------------------------------------------------------- - crypto::public_key generate_multisig_N1_N_spend_public_key(const std::vector<crypto::public_key> &pkeys) + crypto::public_key generate_multisig_M_N_spend_public_key(const std::vector<crypto::public_key> &pkeys) { rct::key spend_public_key = rct::identity(); for (const auto &pk: pkeys) @@ -138,4 +178,9 @@ namespace cryptonote return true; } //----------------------------------------------------------------- + uint32_t multisig_rounds_required(uint32_t participants, uint32_t threshold) + { + CHECK_AND_ASSERT_THROW_MES(participants >= threshold, "participants must be greater or equal than threshold"); + return participants - threshold + 1; + } } diff --git a/src/multisig/multisig.h b/src/multisig/multisig.h index f95611441..93a756812 100644 --- a/src/multisig/multisig.h +++ b/src/multisig/multisig.h @@ -41,9 +41,31 @@ namespace cryptonote crypto::secret_key get_multisig_blinded_secret_key(const crypto::secret_key &key); void generate_multisig_N_N(const account_keys &keys, const std::vector<crypto::public_key> &spend_keys, std::vector<crypto::secret_key> &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey); void generate_multisig_N1_N(const account_keys &keys, const std::vector<crypto::public_key> &spend_keys, std::vector<crypto::secret_key> &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey); + /** + * @brief generate_multisig_derivations performs common DH key derivation. + * Each middle round in M/N scheme is DH exchange of public multisig keys of other participants multiplied by secret spend key of current participant. + * this functions does the following: new multisig key = secret spend * public multisig key + * @param keys - current wallet's keys + * @param derivations - public multisig keys of other participants + * @return new public multisig keys derived from previous round. This data needs to be exchange with other participants + */ + std::vector<crypto::public_key> generate_multisig_derivations(const account_keys &keys, const std::vector<crypto::public_key> &derivations); + crypto::secret_key calculate_multisig_signer_key(const std::vector<crypto::secret_key>& derivations); + /** + * @brief calculate_multisig_keys. Calculates secret multisig keys from others' participants ones as follows: mi = H(Mi) + * @param derivations - others' participants public multisig keys. + * @return vector of current wallet's multisig secret keys + */ + std::vector<crypto::secret_key> calculate_multisig_keys(const std::vector<crypto::public_key>& derivations); crypto::secret_key generate_multisig_view_secret_key(const crypto::secret_key &skey, const std::vector<crypto::secret_key> &skeys); - crypto::public_key generate_multisig_N1_N_spend_public_key(const std::vector<crypto::public_key> &pkeys); + /** + * @brief generate_multisig_M_N_spend_public_key calculates multisig wallet's spend public key by summing all of public multisig keys + * @param pkeys unique public multisig keys + * @return multisig wallet's spend public key + */ + crypto::public_key generate_multisig_M_N_spend_public_key(const std::vector<crypto::public_key> &pkeys); bool generate_multisig_key_image(const account_keys &keys, size_t multisig_key_index, const crypto::public_key& out_key, crypto::key_image& ki); void generate_multisig_LR(const crypto::public_key pkey, const crypto::secret_key &k, crypto::public_key &L, crypto::public_key &R); bool generate_multisig_composite_key_image(const account_keys &keys, const std::unordered_map<crypto::public_key, cryptonote::subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key &tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, const std::vector<crypto::key_image> &pkis, crypto::key_image &ki); + uint32_t multisig_rounds_required(uint32_t participants, uint32_t threshold); } diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt index 83bdffab5..9b924907e 100644 --- a/src/p2p/CMakeLists.txt +++ b/src/p2p/CMakeLists.txt @@ -38,7 +38,6 @@ source_group(p2p FILES ${P2P}) monero_add_library(p2p ${P2P}) target_link_libraries(p2p PUBLIC - epee version cryptonote_core ${UPNP_LIBRARIES} diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index c9ca63f43..e9d2061e8 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -29,6 +29,7 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #include "common/command_line.h" +#include "cryptonote_core/cryptonote_core.h" #include "net_node.h" namespace nodetool diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 4606f66ee..90e2f78b1 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -244,6 +244,7 @@ namespace nodetool bool check_connection_and_handshake_with_peer(const epee::net_utils::network_address& na, uint64_t last_seen_stamp); bool gray_peerlist_housekeeping(); + bool check_incoming_connections(); void kill() { ///< will be called e.g. from deinit() _info("Killing the net_node"); @@ -307,6 +308,7 @@ namespace nodetool epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval; epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval; epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval; + epee::math_helper::once_a_time_seconds<900, false> m_incoming_connections_interval; std::string m_bind_ip; std::string m_port; @@ -316,6 +318,7 @@ namespace nodetool std::list<epee::net_utils::network_address> m_priority_peers; std::vector<epee::net_utils::network_address> m_exclusive_peers; std::vector<epee::net_utils::network_address> m_seed_nodes; + bool m_fallback_seed_nodes_added; std::list<nodetool::peerlist_entry> m_command_line_peers; uint64_t m_peer_livetime; //keep connections to initiate some interactions diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 9b21705ec..a61b6107f 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -62,6 +62,7 @@ namespace nodetool { + inline bool append_net_address(std::vector<epee::net_utils::network_address> & seed_nodes, std::string const & addr, uint16_t default_port); //----------------------------------------------------------------------------------- template<class t_payload_net_handler> void node_server<t_payload_net_handler>::init_options(boost::program_options::options_description& desc) @@ -273,10 +274,22 @@ namespace nodetool { nodetool::peerlist_entry pe = AUTO_VAL_INIT(pe); pe.id = crypto::rand<uint64_t>(); - const uint16_t default_port = testnet ? ::config::testnet::P2P_DEFAULT_PORT : stagenet ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT; + const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; bool r = parse_peer_from_string(pe.adr, pr_str, default_port); - CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str); - m_command_line_peers.push_back(pe); + if (r) + { + m_command_line_peers.push_back(pe); + continue; + } + std::vector<epee::net_utils::network_address> resolved_addrs; + r = append_net_address(resolved_addrs, pr_str, default_port); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); + for (const epee::net_utils::network_address& addr : resolved_addrs) + { + pe.id = crypto::rand<uint64_t>(); + pe.adr = addr; + m_command_line_peers.push_back(pe); + } } } @@ -327,24 +340,31 @@ namespace nodetool return true; } //----------------------------------------------------------------------------------- - inline void append_net_address( + inline bool append_net_address( std::vector<epee::net_utils::network_address> & seed_nodes , std::string const & addr + , uint16_t default_port ) { using namespace boost::asio; + std::string host = addr; + std::string port = std::to_string(default_port); size_t pos = addr.find_last_of(':'); - CHECK_AND_ASSERT_MES_NO_RET(std::string::npos != pos && addr.length() - 1 != pos && 0 != pos, "Failed to parse seed address from string: '" << addr << '\''); - std::string host = addr.substr(0, pos); - std::string port = addr.substr(pos + 1); + if (std::string::npos != pos) + { + CHECK_AND_ASSERT_MES(addr.length() - 1 != pos && 0 != pos, false, "Failed to parse seed address from string: '" << addr << '\''); + host = addr.substr(0, pos); + port = addr.substr(pos + 1); + } + MINFO("Resolving node address: host=" << host << ", port=" << port); io_service io_srv; ip::tcp::resolver resolver(io_srv); ip::tcp::resolver::query query(host, port, boost::asio::ip::tcp::resolver::query::canonical_name); boost::system::error_code ec; ip::tcp::resolver::iterator i = resolver.resolve(query, ec); - CHECK_AND_ASSERT_MES_NO_RET(!ec, "Failed to resolve host name '" << host << "': " << ec.message() << ':' << ec.value()); + CHECK_AND_ASSERT_MES(!ec, false, "Failed to resolve host name '" << host << "': " << ec.message() << ':' << ec.value()); ip::tcp::resolver::iterator iend; for (; i != iend; ++i) @@ -354,14 +374,14 @@ namespace nodetool { epee::net_utils::network_address na{epee::net_utils::ipv4_network_address{boost::asio::detail::socket_ops::host_to_network_long(endpoint.address().to_v4().to_ulong()), endpoint.port()}}; seed_nodes.push_back(na); - MINFO("Added seed node: " << na.str()); + MINFO("Added node: " << na.str()); } else { MWARNING("IPv6 unsupported, skip '" << host << "' -> " << endpoint.address().to_v6().to_string(ec)); - throw std::runtime_error("IPv6 unsupported"); } } + return true; } //----------------------------------------------------------------------------------- @@ -382,6 +402,9 @@ namespace nodetool full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("162.210.173.151:38080"); } + else if (nettype == cryptonote::FAKECHAIN) + { + } else { full_addrs.insert("107.152.130.98:18080"); @@ -405,6 +428,7 @@ namespace nodetool bool res = handle_command_line(vm); CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line"); + m_fallback_seed_nodes_added = false; if (m_nettype == cryptonote::TESTNET) { memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16); @@ -483,7 +507,7 @@ namespace nodetool if (result.size()) { for (const auto& addr_string : result) - full_addrs.insert(addr_string + ":" + std::to_string(m_nettype == cryptonote::TESTNET ? ::config::testnet::P2P_DEFAULT_PORT : m_nettype == cryptonote::STAGENET ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT)); + full_addrs.insert(addr_string + ":" + std::to_string(cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT)); } ++i; } @@ -498,6 +522,7 @@ namespace nodetool for (const auto &peer: get_seed_nodes(cryptonote::MAINNET)) full_addrs.insert(peer); + m_fallback_seed_nodes_added = true; } } } @@ -505,7 +530,7 @@ namespace nodetool for (const auto& full_addr : full_addrs) { MDEBUG("Seed node: " << full_addr); - append_net_address(m_seed_nodes, full_addr); + append_net_address(m_seed_nodes, full_addr, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); } MDEBUG("Number of seed nodes: " << m_seed_nodes.size()); @@ -630,10 +655,14 @@ namespace nodetool { kill(); m_peerlist.deinit(); - m_net_server.deinit_server(); - // remove UPnP port mapping - if(!m_no_igd) - delete_upnp_port_mapping(m_listening_port); + + if (!m_offline) + { + m_net_server.deinit_server(); + // remove UPnP port mapping + if(!m_no_igd) + delete_upnp_port_mapping(m_listening_port); + } return store_config(); } //----------------------------------------------------------------------------------- @@ -1134,7 +1163,6 @@ namespace nodetool size_t try_count = 0; size_t current_index = crypto::rand<size_t>()%m_seed_nodes.size(); - bool fallback_nodes_added = false; while(true) { if(m_net_server.is_stop_signal_sent()) @@ -1144,15 +1172,21 @@ namespace nodetool break; if(++try_count > m_seed_nodes.size()) { - if (!fallback_nodes_added) + if (!m_fallback_seed_nodes_added) { MWARNING("Failed to connect to any of seed peers, trying fallback seeds"); + current_index = m_seed_nodes.size(); for (const auto &peer: get_seed_nodes(m_nettype)) { MDEBUG("Fallback seed node: " << peer); - append_net_address(m_seed_nodes, peer); + append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); + } + m_fallback_seed_nodes_added = true; + if (current_index == m_seed_nodes.size()) + { + MWARNING("No fallback seeds, continuing without seeds"); + break; } - fallback_nodes_added = true; // continue for another few cycles } else @@ -1293,6 +1327,20 @@ namespace nodetool m_connections_maker_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::connections_maker, this)); m_gray_peerlist_housekeeping_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::gray_peerlist_housekeeping, this)); m_peerlist_store_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::store_config, this)); + m_incoming_connections_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::check_incoming_connections, this)); + return true; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::check_incoming_connections() + { + if (m_offline || m_hide_my_port) + return true; + if (get_incoming_connections_count() == 0) + { + const el::Level level = el::Level::Warning; + MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + } return true; } //----------------------------------------------------------------------------------- @@ -1807,10 +1855,20 @@ namespace nodetool for(const std::string& pr_str: perrs) { epee::net_utils::network_address na = AUTO_VAL_INIT(na); - const uint16_t default_port = m_nettype == cryptonote::TESTNET ? ::config::testnet::P2P_DEFAULT_PORT : m_nettype == cryptonote::STAGENET ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT; + const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; bool r = parse_peer_from_string(na, pr_str, default_port); - CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str); - container.push_back(na); + if (r) + { + container.push_back(na); + continue; + } + std::vector<epee::net_utils::network_address> resolved_addrs; + r = append_net_address(resolved_addrs, pr_str, default_port); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); + for (const epee::net_utils::network_address& addr : resolved_addrs) + { + container.push_back(addr); + } } return true; @@ -1988,7 +2046,7 @@ namespace nodetool char lanAddress[64]; result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); freeUPNPDevlist(deviceList); - if (result != 0) { + if (result > 0) { if (result == 1) { std::ostringstream portString; portString << port; @@ -2034,7 +2092,7 @@ namespace nodetool char lanAddress[64]; result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); freeUPNPDevlist(deviceList); - if (result != 0) { + if (result > 0) { if (result == 1) { std::ostringstream portString; portString << port; diff --git a/src/ringct/CMakeLists.txt b/src/ringct/CMakeLists.txt index c8dcdca26..29f166a3b 100644 --- a/src/ringct/CMakeLists.txt +++ b/src/ringct/CMakeLists.txt @@ -30,11 +30,13 @@ set(ringct_basic_sources rctOps.cpp rctTypes.cpp rctCryptoOps.c + multiexp.cc bulletproofs.cc) set(ringct_basic_private_headers rctOps.h rctTypes.h + multiexp.h bulletproofs.h) monero_private_headers(ringct_basic diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index fd15ffbc4..bed48769a 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -29,15 +29,16 @@ // Adapted from Java code by Sarang Noether #include <stdlib.h> -#include <openssl/ssl.h> #include <boost/thread/mutex.hpp> #include "misc_log_ex.h" #include "common/perf_timer.h" +#include "cryptonote_config.h" extern "C" { #include "crypto/crypto-ops.h" } #include "rctOps.h" +#include "multiexp.h" #include "bulletproofs.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -45,29 +46,62 @@ extern "C" //#define DEBUG_BP +#if 1 #define PERF_TIMER_START_BP(x) PERF_TIMER_START_UNIT(x, 1000000) +#define PERF_TIMER_STOP_BP(x) PERF_TIMER_STOP(x) +#else +#define PERF_TIMER_START_BP(x) ((void*)0) +#define PERF_TIMER_STOP_BP(x) ((void*)0) +#endif + +#define STRAUS_SIZE_LIMIT 232 +#define PIPPENGER_SIZE_LIMIT 0 namespace rct { static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b); -static rct::keyV vector_powers(rct::key x, size_t n); +static rct::keyV vector_powers(const rct::key &x, size_t n); +static rct::keyV vector_dup(const rct::key &x, size_t n); static rct::key inner_product(const rct::keyV &a, const rct::keyV &b); static constexpr size_t maxN = 64; -static rct::key Hi[maxN], Gi[maxN]; -static ge_dsmp Gprecomp[64], Hprecomp[64]; +static constexpr size_t maxM = BULLETPROOF_MAX_OUTPUTS; +static rct::key Hi[maxN*maxM], Gi[maxN*maxM]; +static ge_p3 Hi_p3[maxN*maxM], Gi_p3[maxN*maxM]; +static std::shared_ptr<straus_cached_data> straus_HiGi_cache; +static std::shared_ptr<pippenger_cached_data> pippenger_HiGi_cache; static const rct::key TWO = { {0x02, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; -static const rct::keyV oneN = vector_powers(rct::identity(), maxN); +static const rct::key MINUS_ONE = { { 0xec, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; +static const rct::key MINUS_INV_EIGHT = { { 0x74, 0xa4, 0x19, 0x7a, 0xf0, 0x7d, 0x0b, 0xf7, 0x05, 0xc2, 0xda, 0x25, 0x2b, 0x5c, 0x0b, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a } }; +static const rct::keyV oneN = vector_dup(rct::identity(), maxN); static const rct::keyV twoN = vector_powers(TWO, maxN); static const rct::key ip12 = inner_product(oneN, twoN); static boost::mutex init_mutex; +static inline rct::key multiexp(const std::vector<MultiexpData> &data, size_t HiGi_size) +{ + if (HiGi_size > 0) + { + static_assert(232 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); + return HiGi_size <= 232 && data.size() == HiGi_size ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, HiGi_size, get_pippenger_c(data.size())); + } + else + return data.size() <= 95 ? straus(data, NULL, 0) : pippenger(data, NULL, 0, get_pippenger_c(data.size())); +} + +static inline bool is_reduced(const rct::key &scalar) +{ + return sc_check(scalar.bytes) == 0; +} + static rct::key get_exponent(const rct::key &base, size_t idx) { static const std::string salt("bulletproof"); std::string hashed = std::string((const char*)base.bytes, sizeof(base)) + salt + tools::get_varint_data(idx); - return rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + const rct::key e = rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + CHECK_AND_ASSERT_THROW_MES(!(e == rct::identity()), "Exponent is point at infinity"); + return e; } static void init_exponents() @@ -77,13 +111,27 @@ static void init_exponents() static bool init_done = false; if (init_done) return; - for (size_t i = 0; i < maxN; ++i) + std::vector<MultiexpData> data; + for (size_t i = 0; i < maxN*maxM; ++i) { Hi[i] = get_exponent(rct::H, i * 2); - rct::precomp(Hprecomp[i], Hi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Hi_p3[i], Hi[i].bytes) == 0, "ge_frombytes_vartime failed"); Gi[i] = get_exponent(rct::H, i * 2 + 1); - rct::precomp(Gprecomp[i], Gi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Gi_p3[i], Gi[i].bytes) == 0, "ge_frombytes_vartime failed"); + + data.push_back({rct::zero(), Gi_p3[i]}); + data.push_back({rct::zero(), Hi_p3[i]}); } + + straus_HiGi_cache = straus_init_cache(data, STRAUS_SIZE_LIMIT); + pippenger_HiGi_cache = pippenger_init_cache(data, 0, PIPPENGER_SIZE_LIMIT); + + MINFO("Hi/Gi cache size: " << (sizeof(Hi)+sizeof(Gi))/1024 << " kB"); + MINFO("Hi_p3/Gi_p3 cache size: " << (sizeof(Hi_p3)+sizeof(Gi_p3))/1024 << " kB"); + MINFO("Straus cache size: " << straus_get_cache_size(straus_HiGi_cache)/1024 << " kB"); + MINFO("Pippenger cache size: " << pippenger_get_cache_size(pippenger_HiGi_cache)/1024 << " kB"); + size_t cache_size = (sizeof(Hi)+sizeof(Hi_p3))*2 + straus_get_cache_size(straus_HiGi_cache) + pippenger_get_cache_size(pippenger_HiGi_cache); + MINFO("Total cache size: " << cache_size/1024 << "kB"); init_done = true; } @@ -91,61 +139,50 @@ static void init_exponents() static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; - rct::addKeys3(term, a[i], Gprecomp[i], b[i], Hprecomp[i]); - rct::addKeys(res, res, term); + multiexp_data.emplace_back(a[i], Gi_p3[i]); + multiexp_data.emplace_back(b[i], Hi_p3[i]); } - return res; + return multiexp(multiexp_data, 2 * a.size()); } /* Compute a custom vector-scalar commitment */ -static rct::key vector_exponent_custom(const rct::keyV &A, const rct::keyV &B, const rct::keyV &a, const rct::keyV &b) +static rct::key cross_vector_exponent8(size_t size, const std::vector<ge_p3> &A, size_t Ao, const std::vector<ge_p3> &B, size_t Bo, const rct::keyV &a, size_t ao, const rct::keyV &b, size_t bo, const rct::keyV *scale, const ge_p3 *extra_point, const rct::key *extra_scalar) { - CHECK_AND_ASSERT_THROW_MES(A.size() == B.size(), "Incompatible sizes of A and B"); - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() == A.size(), "Incompatible sizes of a and A"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); - for (size_t i = 0; i < a.size(); ++i) + CHECK_AND_ASSERT_THROW_MES(size + Ao <= A.size(), "Incompatible size for A"); + CHECK_AND_ASSERT_THROW_MES(size + Bo <= B.size(), "Incompatible size for B"); + CHECK_AND_ASSERT_THROW_MES(size + ao <= a.size(), "Incompatible size for a"); + CHECK_AND_ASSERT_THROW_MES(size + bo <= b.size(), "Incompatible size for b"); + CHECK_AND_ASSERT_THROW_MES(size <= maxN*maxM, "size is too large"); + CHECK_AND_ASSERT_THROW_MES(!scale || size == scale->size() / 2, "Incompatible size for scale"); + CHECK_AND_ASSERT_THROW_MES(!!extra_point == !!extra_scalar, "only one of extra point/scalar present"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.resize(size*2 + (!!extra_point)); + for (size_t i = 0; i < size; ++i) { - rct::key term; -#if 0 - // we happen to know where A and B might fall, so don't bother checking the rest - ge_dsmp *Acache = NULL, *Bcache = NULL; - ge_dsmp Acache_custom[1], Bcache_custom[1]; - if (Gi[i] == A[i]) - Acache = Gprecomp + i; - else if (i<32 && Gi[i+32] == A[i]) - Acache = Gprecomp + i + 32; - else - { - rct::precomp(Acache_custom[0], A[i]); - Acache = Acache_custom; - } - if (i == 0 && B[i] == Hi[0]) - Bcache = Hprecomp; - else - { - rct::precomp(Bcache_custom[0], B[i]); - Bcache = Bcache_custom; - } - rct::addKeys3(term, a[i], *Acache, b[i], *Bcache); -#else - ge_dsmp Acache, Bcache; - rct::precomp(Bcache, B[i]); - rct::addKeys3(term, a[i], A[i], b[i], Bcache); -#endif - rct::addKeys(res, res, term); + sc_mul(multiexp_data[i*2].scalar.bytes, a[ao+i].bytes, INV_EIGHT.bytes);; + multiexp_data[i*2].point = A[Ao+i]; + sc_mul(multiexp_data[i*2+1].scalar.bytes, b[bo+i].bytes, INV_EIGHT.bytes); + if (scale) + sc_mul(multiexp_data[i*2+1].scalar.bytes, multiexp_data[i*2+1].scalar.bytes, (*scale)[Bo+i].bytes); + multiexp_data[i*2+1].point = B[Bo+i]; } - return res; + if (extra_point) + { + sc_mul(multiexp_data.back().scalar.bytes, extra_scalar->bytes, INV_EIGHT.bytes); + multiexp_data.back().point = *extra_point; + } + return multiexp(multiexp_data, 0); } /* Given a scalar, construct a vector of powers */ -static rct::keyV vector_powers(rct::key x, size_t n) +static rct::keyV vector_powers(const rct::key &x, size_t n) { rct::keyV res(n); if (n == 0) @@ -161,6 +198,24 @@ static rct::keyV vector_powers(rct::key x, size_t n) return res; } +/* Given a scalar, return the sum of its powers from 0 to n-1 */ +static rct::key vector_power_sum(const rct::key &x, size_t n) +{ + if (n == 0) + return rct::zero(); + rct::key res = rct::identity(); + if (n == 1) + return res; + rct::key prev = x; + for (size_t i = 1; i < n; ++i) + { + if (i > 1) + sc_mul(prev.bytes, prev.bytes, x.bytes); + sc_add(res.bytes, res.bytes, prev.bytes); + } + return res; +} + /* Given two scalar arrays, construct the inner product */ static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) { @@ -185,16 +240,22 @@ static rct::keyV hadamard(const rct::keyV &a, const rct::keyV &b) return res; } -/* Given two curvepoint arrays, construct the Hadamard product */ -static rct::keyV hadamard2(const rct::keyV &a, const rct::keyV &b) +/* folds a curvepoint array using a two way scaled Hadamard product */ +static void hadamard_fold(std::vector<ge_p3> &v, const rct::keyV *scale, const rct::key &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - rct::keyV res(a.size()); - for (size_t i = 0; i < a.size(); ++i) + CHECK_AND_ASSERT_THROW_MES((v.size() & 1) == 0, "Vector size should be even"); + const size_t sz = v.size() / 2; + for (size_t n = 0; n < sz; ++n) { - rct::addKeys(res[i], a[i], b[i]); + ge_dsmp c[2]; + ge_dsm_precomp(c[0], &v[n]); + ge_dsm_precomp(c[1], &v[sz + n]); + rct::key sa, sb; + if (scale) sc_mul(sa.bytes, a.bytes, (*scale)[n].bytes); else sa = a; + if (scale) sc_mul(sb.bytes, b.bytes, (*scale)[sz + n].bytes); else sb = b; + ge_double_scalarmult_precomp_vartime2_p3(&v[n], sa.bytes, c[0], sb.bytes, c[1]); } - return res; + v.resize(sz); } /* Add two vectors */ @@ -209,71 +270,98 @@ static rct::keyV vector_add(const rct::keyV &a, const rct::keyV &b) return res; } -/* Subtract two vectors */ -static rct::keyV vector_subtract(const rct::keyV &a, const rct::keyV &b) +/* Add a scalar to all elements of a vector */ +static rct::keyV vector_add(const rct::keyV &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_sub(res[i].bytes, a[i].bytes, b[i].bytes); + sc_add(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Multiply a scalar and a vector */ -static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) +/* Subtract a scalar from all elements of a vector */ +static rct::keyV vector_subtract(const rct::keyV &a, const rct::key &b) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_mul(res[i].bytes, a[i].bytes, x.bytes); + sc_sub(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Exponentiate a curve vector by a scalar */ -static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) +/* Multiply a scalar and a vector */ +static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - rct::scalarmultKey(res[i], a[i], x); + sc_mul(res[i].bytes, a[i].bytes, x.bytes); } return res; } -static rct::key switch_endianness(rct::key k) +/* Create a vector from copies of a single value */ +static rct::keyV vector_dup(const rct::key &x, size_t N) { - std::reverse(k.bytes, k.bytes + sizeof(k)); - return k; + return rct::keyV(N, x); } -/* Compute the inverse of a scalar, the stupid way */ -static rct::key invert(const rct::key &x) +static rct::key sm(rct::key y, int n, const rct::key &x) { - rct::key inv; - - BN_CTX *ctx = BN_CTX_new(); - BIGNUM *X = BN_new(); - BIGNUM *L = BN_new(); - BIGNUM *I = BN_new(); - - BN_bin2bn(switch_endianness(x).bytes, sizeof(rct::key), X); - BN_bin2bn(switch_endianness(rct::curveOrder()).bytes, sizeof(rct::key), L); - - CHECK_AND_ASSERT_THROW_MES(BN_mod_inverse(I, X, L, ctx), "Failed to invert"); + while (n--) + sc_mul(y.bytes, y.bytes, y.bytes); + sc_mul(y.bytes, y.bytes, x.bytes); + return y; +} - const int len = BN_num_bytes(I); - CHECK_AND_ASSERT_THROW_MES((size_t)len <= sizeof(rct::key), "Invalid number length"); - inv = rct::zero(); - BN_bn2bin(I, inv.bytes); - std::reverse(inv.bytes, inv.bytes + len); +/* Compute the inverse of a scalar, the clever way */ +static rct::key invert(const rct::key &x) +{ + rct::key _1, _10, _100, _11, _101, _111, _1001, _1011, _1111; + + _1 = x; + sc_mul(_10.bytes, _1.bytes, _1.bytes); + sc_mul(_100.bytes, _10.bytes, _10.bytes); + sc_mul(_11.bytes, _10.bytes, _1.bytes); + sc_mul(_101.bytes, _10.bytes, _11.bytes); + sc_mul(_111.bytes, _10.bytes, _101.bytes); + sc_mul(_1001.bytes, _10.bytes, _111.bytes); + sc_mul(_1011.bytes, _10.bytes, _1001.bytes); + sc_mul(_1111.bytes, _100.bytes, _1011.bytes); - BN_free(I); - BN_free(L); - BN_free(X); - BN_CTX_free(ctx); + rct::key inv; + sc_mul(inv.bytes, _1111.bytes, _1.bytes); + + inv = sm(inv, 123 + 3, _101); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 4, _1001); + inv = sm(inv, 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 3, _101); + inv = sm(inv, 3 + 3, _101); + inv = sm(inv, 3, _111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 2 + 3, _111); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 2 + 4, _1011); + inv = sm(inv, 6 + 4, _1001); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 1 + 4, _1001); + inv = sm(inv, 1 + 3, _111); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 3, _101); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 3, _101); + inv = sm(inv, 1 + 2, _11); #ifdef DEBUG_BP rct::key tmp; @@ -283,6 +371,34 @@ static rct::key invert(const rct::key &x) return inv; } +static rct::keyV invert(rct::keyV x) +{ + rct::keyV scratch; + scratch.reserve(x.size()); + + rct::key acc = rct::identity(); + for (size_t n = 0; n < x.size(); ++n) + { + scratch.push_back(acc); + if (n == 0) + acc = x[0]; + else + sc_mul(acc.bytes, acc.bytes, x[n].bytes); + } + + acc = invert(acc); + + rct::key tmp; + for (int i = x.size(); i-- > 0; ) + { + sc_mul(tmp.bytes, acc.bytes, x[i].bytes); + sc_mul(x[i].bytes, acc.bytes, scratch[i].bytes); + acc = tmp; + } + + return x; +} + /* Compute the slice of a vector */ static rct::keyV slice(const rct::keyV &a, size_t start, size_t stop) { @@ -333,147 +449,207 @@ static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, con /* Given a value v (0..2^N-1) and a mask gamma, construct a range proof */ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) { + return bulletproof_PROVE(rct::keyV(1, sv), rct::keyV(1, gamma)); +} + +Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) +{ + return bulletproof_PROVE(std::vector<uint64_t>(1, v), rct::keyV(1, gamma)); +} + +/* Given a set of values v (0..2^N-1) and masks gamma, construct a range proof */ +Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) +{ + CHECK_AND_ASSERT_THROW_MES(sv.size() == gamma.size(), "Incompatible sizes of sv and gamma"); + CHECK_AND_ASSERT_THROW_MES(!sv.empty(), "sv is empty"); + for (const rct::key &sve: sv) + CHECK_AND_ASSERT_THROW_MES(is_reduced(sve), "Invalid sv input"); + for (const rct::key &g: gamma) + CHECK_AND_ASSERT_THROW_MES(is_reduced(g), "Invalid gamma input"); + init_exponents(); PERF_TIMER_UNIT(PROVE, 1000000); constexpr size_t logN = 6; // log2(64) constexpr size_t N = 1<<logN; - - rct::key V; - rct::keyV aL(N), aR(N); + size_t M, logM; + for (logM = 0; (M = 1<<logM) <= maxM && M < sv.size(); ++logM); + CHECK_AND_ASSERT_THROW_MES(M <= maxM, "sv/gamma are too large"); + const size_t logMN = logM + logN; + const size_t MN = M * N; + + rct::keyV V(sv.size()); + rct::keyV aL(MN), aR(MN); + rct::keyV aL8(MN), aR8(MN); + rct::key tmp, tmp2; PERF_TIMER_START_BP(PROVE_v); - rct::addKeys2(V, gamma, sv, rct::H); - PERF_TIMER_STOP(PROVE_v); + for (size_t i = 0; i < sv.size(); ++i) + { + rct::key gamma8, sv8; + sc_mul(gamma8.bytes, gamma[i].bytes, INV_EIGHT.bytes); + sc_mul(sv8.bytes, sv[i].bytes, INV_EIGHT.bytes); + rct::addKeys2(V[i], gamma8, sv8, rct::H); + } + PERF_TIMER_STOP_BP(PROVE_v); PERF_TIMER_START_BP(PROVE_aLaR); - for (size_t i = N; i-- > 0; ) + for (size_t j = 0; j < M; ++j) { - if (sv[i/8] & (((uint64_t)1)<<(i%8))) - { - aL[i] = rct::identity(); - } - else + for (size_t i = N; i-- > 0; ) { - aL[i] = rct::zero(); + if (j < sv.size() && (sv[j][i/8] & (((uint64_t)1)<<(i%8)))) + { + aL[j*N+i] = rct::identity(); + aL8[j*N+i] = INV_EIGHT; + aR[j*N+i] = aR8[j*N+i] = rct::zero(); + } + else + { + aL[j*N+i] = aL8[j*N+i] = rct::zero(); + aR[j*N+i] = MINUS_ONE; + aR8[j*N+i] = MINUS_INV_EIGHT; + } } - sc_sub(aR[i].bytes, aL[i].bytes, rct::identity().bytes); } - PERF_TIMER_STOP(PROVE_aLaR); - - rct::key hash_cache = rct::hash_to_scalar(V); + PERF_TIMER_STOP_BP(PROVE_aLaR); // DEBUG: Test to ensure this recovers the value #ifdef DEBUG_BP - uint64_t test_aL = 0, test_aR = 0; - for (size_t i = 0; i < N; ++i) - { - if (aL[i] == rct::identity()) - test_aL += ((uint64_t)1)<<i; - if (aR[i] == rct::zero()) - test_aR += ((uint64_t)1)<<i; - } - uint64_t v_test = 0; - for (int n = 0; n < 8; ++n) v_test |= (((uint64_t)sv[n]) << (8*n)); - CHECK_AND_ASSERT_THROW_MES(test_aL == v_test, "test_aL failed"); - CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); + for (size_t j = 0; j < M; ++j) + { + uint64_t test_aL = 0, test_aR = 0; + for (size_t i = 0; i < N; ++i) + { + if (aL[j*N+i] == rct::identity()) + test_aL += ((uint64_t)1)<<i; + if (aR[j*N+i] == rct::zero()) + test_aR += ((uint64_t)1)<<i; + } + uint64_t v_test = 0; + if (j < sv.size()) + for (int n = 0; n < 8; ++n) v_test |= (((uint64_t)sv[j][n]) << (8*n)); + CHECK_AND_ASSERT_THROW_MES(test_aL == v_test, "test_aL failed"); + CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); + } #endif +try_again: + rct::key hash_cache = rct::hash_to_scalar(V); + PERF_TIMER_START_BP(PROVE_step1); // PAPER LINES 38-39 rct::key alpha = rct::skGen(); - rct::key ve = vector_exponent(aL, aR); + rct::key ve = vector_exponent(aL8, aR8); rct::key A; - rct::addKeys(A, ve, rct::scalarmultBase(alpha)); + sc_mul(tmp.bytes, alpha.bytes, INV_EIGHT.bytes); + rct::addKeys(A, ve, rct::scalarmultBase(tmp)); // PAPER LINES 40-42 - rct::keyV sL = rct::skvGen(N), sR = rct::skvGen(N); + rct::keyV sL = rct::skvGen(MN), sR = rct::skvGen(MN); rct::key rho = rct::skGen(); ve = vector_exponent(sL, sR); rct::key S; rct::addKeys(S, ve, rct::scalarmultBase(rho)); + S = rct::scalarmultKey(S, INV_EIGHT); // PAPER LINES 43-45 rct::key y = hash_cache_mash(hash_cache, A, S); + if (y == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step1); + MINFO("y is 0, trying again"); + goto try_again; + } rct::key z = hash_cache = rct::hash_to_scalar(y); + if (z == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step1); + MINFO("z is 0, trying again"); + goto try_again; + } - // Polynomial construction before PAPER LINE 46 - rct::key t0 = rct::zero(); - rct::key t1 = rct::zero(); - rct::key t2 = rct::zero(); - - const auto yN = vector_powers(y, N); - - rct::key ip1y = inner_product(oneN, yN); - rct::key tmp; - sc_muladd(t0.bytes, z.bytes, ip1y.bytes, t0.bytes); + // Polynomial construction by coefficients + rct::keyV l0 = vector_subtract(aL, z); + const rct::keyV &l1 = sL; - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - sc_muladd(t0.bytes, zsq.bytes, sv.bytes, t0.bytes); + // This computes the ugly sum/concatenation from PAPER LINE 65 + rct::keyV zero_twos(MN); + const rct::keyV zpow = vector_powers(z, M+2); + for (size_t i = 0; i < MN; ++i) + { + zero_twos[i] = rct::zero(); + for (size_t j = 1; j <= M; ++j) + { + if (i >= (j-1)*N && i < j*N) + { + CHECK_AND_ASSERT_THROW_MES(1+j < zpow.size(), "invalid zpow index"); + CHECK_AND_ASSERT_THROW_MES(i-(j-1)*N < twoN.size(), "invalid twoN index"); + sc_muladd(zero_twos[i].bytes, zpow[1+j].bytes, twoN[i-(j-1)*N].bytes, zero_twos[i].bytes); + } + } + } - rct::key k = rct::zero(); - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); + rct::keyV r0 = vector_add(aR, z); + const auto yMN = vector_powers(y, MN); + r0 = hadamard(r0, yMN); + r0 = vector_add(r0, zero_twos); + rct::keyV r1 = hadamard(yMN, sR); - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - sc_add(t0.bytes, t0.bytes, k.bytes); + // Polynomial construction before PAPER LINE 46 + rct::key t1_1 = inner_product(l0, r1); + rct::key t1_2 = inner_product(l1, r0); + rct::key t1; + sc_add(t1.bytes, t1_1.bytes, t1_2.bytes); + rct::key t2 = inner_product(l1, r1); - // DEBUG: Test the value of t0 has the correct form -#ifdef DEBUG_BP - rct::key test_t0 = rct::zero(); - rct::key iph = inner_product(aL, hadamard(aR, yN)); - sc_add(test_t0.bytes, test_t0.bytes, iph.bytes); - rct::key ips = inner_product(vector_subtract(aL, aR), yN); - sc_muladd(test_t0.bytes, z.bytes, ips.bytes, test_t0.bytes); - rct::key ipt = inner_product(twoN, aL); - sc_muladd(test_t0.bytes, zsq.bytes, ipt.bytes, test_t0.bytes); - sc_add(test_t0.bytes, test_t0.bytes, k.bytes); - CHECK_AND_ASSERT_THROW_MES(t0 == test_t0, "t0 check failed"); -#endif - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); PERF_TIMER_START_BP(PROVE_step2); - const auto HyNsR = hadamard(yN, sR); - const auto vpIz = vector_scalar(oneN, z); - const auto vp2zsq = vector_scalar(twoN, zsq); - const auto aL_vpIz = vector_subtract(aL, vpIz); - const auto aR_vpIz = vector_add(aR, vpIz); - - rct::key ip1 = inner_product(aL_vpIz, HyNsR); - sc_add(t1.bytes, t1.bytes, ip1.bytes); - - rct::key ip2 = inner_product(sL, vector_add(hadamard(yN, aR_vpIz), vp2zsq)); - sc_add(t1.bytes, t1.bytes, ip2.bytes); - - rct::key ip3 = inner_product(sL, HyNsR); - sc_add(t2.bytes, t2.bytes, ip3.bytes); - // PAPER LINES 47-48 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - rct::key T1 = rct::addKeys(rct::scalarmultKey(rct::H, t1), rct::scalarmultBase(tau1)); - rct::key T2 = rct::addKeys(rct::scalarmultKey(rct::H, t2), rct::scalarmultBase(tau2)); + rct::key T1, T2; + ge_p3 p3; + sc_mul(tmp.bytes, t1.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau1.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T1.bytes, &p3); + sc_mul(tmp.bytes, t2.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau2.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T2.bytes, &p3); // PAPER LINES 49-51 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } // PAPER LINES 52-53 - rct::key taux = rct::zero(); + rct::key taux; sc_mul(taux.bytes, tau1.bytes, x.bytes); rct::key xsq; sc_mul(xsq.bytes, x.bytes, x.bytes); sc_muladd(taux.bytes, tau2.bytes, xsq.bytes, taux.bytes); - sc_muladd(taux.bytes, gamma.bytes, zsq.bytes, taux.bytes); + for (size_t j = 1; j <= sv.size(); ++j) + { + CHECK_AND_ASSERT_THROW_MES(j+1 < zpow.size(), "invalid zpow index"); + sc_muladd(taux.bytes, zpow[j+1].bytes, gamma[j-1].bytes, taux.bytes); + } rct::key mu; sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); // PAPER LINES 54-57 - rct::keyV l = vector_add(aL_vpIz, vector_scalar(sL, x)); - rct::keyV r = vector_add(hadamard(yN, vector_add(aR_vpIz, vector_scalar(sR, x))), vp2zsq); - PERF_TIMER_STOP(PROVE_step2); + rct::keyV l = l0; + l = vector_add(l, vector_scalar(l1, x)); + rct::keyV r = r0; + r = vector_add(r, vector_scalar(r1, x)); + PERF_TIMER_STOP_BP(PROVE_step2); PERF_TIMER_START_BP(PROVE_step3); rct::key t = inner_product(l, r); @@ -481,6 +657,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // DEBUG: Test if the l and r vectors match the polynomial forms #ifdef DEBUG_BP rct::key test_t; + const rct::key t0 = inner_product(l0, r0); sc_muladd(test_t.bytes, t1.bytes, x.bytes, t0.bytes); sc_muladd(test_t.bytes, t2.bytes, xsq.bytes, test_t.bytes); CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); @@ -488,265 +665,407 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 32-33 rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); + if (x_ip == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step3); + MINFO("x_ip is 0, trying again"); + goto try_again; + } // These are used in the inner product rounds - size_t nprime = N; - rct::keyV Gprime(N); - rct::keyV Hprime(N); - rct::keyV aprime(N); - rct::keyV bprime(N); + size_t nprime = MN; + std::vector<ge_p3> Gprime(MN); + std::vector<ge_p3> Hprime(MN); + rct::keyV aprime(MN); + rct::keyV bprime(MN); const rct::key yinv = invert(y); - rct::key yinvpow = rct::identity(); - for (size_t i = 0; i < N; ++i) + rct::keyV yinvpow(MN); + yinvpow[0] = rct::identity(); + yinvpow[1] = yinv; + for (size_t i = 0; i < MN; ++i) { - Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi[i], yinvpow); - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + Gprime[i] = Gi_p3[i]; + Hprime[i] = Hi_p3[i]; + if (i > 1) + sc_mul(yinvpow[i].bytes, yinvpow[i-1].bytes, yinv.bytes); aprime[i] = l[i]; bprime[i] = r[i]; } - rct::keyV L(logN); - rct::keyV R(logN); + rct::keyV L(logMN); + rct::keyV R(logMN); int round = 0; - rct::keyV w(logN); // this is the challenge x in the inner product protocol - PERF_TIMER_STOP(PROVE_step3); + rct::keyV w(logMN); // this is the challenge x in the inner product protocol + PERF_TIMER_STOP_BP(PROVE_step3); PERF_TIMER_START_BP(PROVE_step4); // PAPER LINE 13 + const rct::keyV *scale = &yinvpow; while (nprime > 1) { // PAPER LINE 15 nprime /= 2; // PAPER LINES 16-17 + PERF_TIMER_START_BP(PROVE_inner_product); rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + PERF_TIMER_STOP_BP(PROVE_inner_product); // PAPER LINES 18-19 - L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + PERF_TIMER_START_BP(PROVE_LR); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultKey(rct::H, tmp)); - R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + L[round] = cross_vector_exponent8(nprime, Gprime, nprime, Hprime, 0, aprime, 0, bprime, nprime, scale, &ge_p3_H, &tmp); sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultKey(rct::H, tmp)); + R[round] = cross_vector_exponent8(nprime, Gprime, 0, Hprime, nprime, aprime, nprime, bprime, 0, scale, &ge_p3_H, &tmp); + PERF_TIMER_STOP_BP(PROVE_LR); // PAPER LINES 21-22 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP_BP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } // PAPER LINES 24-25 const rct::key winv = invert(w[round]); - Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); - Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); + if (nprime > 1) + { + PERF_TIMER_START_BP(PROVE_hadamard2); + hadamard_fold(Gprime, NULL, winv, w[round]); + hadamard_fold(Hprime, scale, w[round], winv); + PERF_TIMER_STOP_BP(PROVE_hadamard2); + } // PAPER LINES 28-29 + PERF_TIMER_START_BP(PROVE_prime); aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); + PERF_TIMER_STOP_BP(PROVE_prime); + scale = NULL; ++round; } - PERF_TIMER_STOP(PROVE_step4); + PERF_TIMER_STOP_BP(PROVE_step4); // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) - return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); + return Bulletproof(std::move(V), A, S, T1, T2, taux, mu, std::move(L), std::move(R), aprime[0], bprime[0], t); } -Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma) { + CHECK_AND_ASSERT_THROW_MES(v.size() == gamma.size(), "Incompatible sizes of v and gamma"); + // vG + gammaH PERF_TIMER_START_BP(PROVE_v); - rct::key sv = rct::zero(); - sv.bytes[0] = v & 255; - sv.bytes[1] = (v >> 8) & 255; - sv.bytes[2] = (v >> 16) & 255; - sv.bytes[3] = (v >> 24) & 255; - sv.bytes[4] = (v >> 32) & 255; - sv.bytes[5] = (v >> 40) & 255; - sv.bytes[6] = (v >> 48) & 255; - sv.bytes[7] = (v >> 56) & 255; - PERF_TIMER_STOP(PROVE_v); + rct::keyV sv(v.size()); + for (size_t i = 0; i < v.size(); ++i) + { + sv[i] = rct::zero(); + sv[i].bytes[0] = v[i] & 255; + sv[i].bytes[1] = (v[i] >> 8) & 255; + sv[i].bytes[2] = (v[i] >> 16) & 255; + sv[i].bytes[3] = (v[i] >> 24) & 255; + sv[i].bytes[4] = (v[i] >> 32) & 255; + sv[i].bytes[5] = (v[i] >> 40) & 255; + sv[i].bytes[6] = (v[i] >> 48) & 255; + sv[i].bytes[7] = (v[i] >> 56) & 255; + } + PERF_TIMER_STOP_BP(PROVE_v); return bulletproof_PROVE(sv, gamma); } +struct proof_data_t +{ + rct::key x, y, z, x_ip; + std::vector<rct::key> w; + size_t logM, inv_offset; +}; + /* Given a range proof, determine if it is valid */ -bool bulletproof_VERIFY(const Bulletproof &proof) +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) { init_exponents(); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "V does not have exactly one element"); - CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); - CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - CHECK_AND_ASSERT_MES(proof.L.size() == 6, false, "Proof is not for 64 bits"); - - const size_t logN = proof.L.size(); - const size_t N = 1 << logN; - - // Reconstruct the challenges PERF_TIMER_START_BP(VERIFY); - PERF_TIMER_START_BP(VERIFY_start); - rct::key hash_cache = rct::hash_to_scalar(proof.V[0]); - rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); - rct::key z = hash_cache = rct::hash_to_scalar(y); - rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); - PERF_TIMER_STOP(VERIFY_start); - - PERF_TIMER_START_BP(VERIFY_line_60); - // Reconstruct the challenges - rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); - PERF_TIMER_STOP(VERIFY_line_60); - - PERF_TIMER_START_BP(VERIFY_line_61); - // PAPER LINE 61 - rct::key L61Left = rct::addKeys(rct::scalarmultBase(proof.taux), rct::scalarmultKey(rct::H, proof.t)); - - rct::key k = rct::zero(); - const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - rct::key tmp, tmp2; - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - PERF_TIMER_STOP(VERIFY_line_61); - - PERF_TIMER_START_BP(VERIFY_line_61rl); - sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); - rct::key L61Right = rct::scalarmultKey(rct::H, tmp); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "proof.V does not have exactly one element"); - tmp = rct::scalarmultKey(proof.V[0], zsq); - rct::addKeys(L61Right, L61Right, tmp); - - tmp = rct::scalarmultKey(proof.T1, x); - rct::addKeys(L61Right, L61Right, tmp); - - rct::key xsq; - sc_mul(xsq.bytes, x.bytes, x.bytes); - tmp = rct::scalarmultKey(proof.T2, xsq); - rct::addKeys(L61Right, L61Right, tmp); - PERF_TIMER_STOP(VERIFY_line_61rl); + const size_t logN = 6; + const size_t N = 1 << logN; - if (!(L61Right == L61Left)) + // sanity and figure out which proof is longest + size_t max_length = 0; + size_t nV = 0; + std::vector<proof_data_t> proof_data; + proof_data.reserve(proofs.size()); + size_t inv_offset = 0; + std::vector<rct::key> to_invert; + to_invert.reserve(11 * sizeof(proofs)); + for (const Bulletproof *p: proofs) { - MERROR("Verification failure at step 1"); - return false; - } - - PERF_TIMER_START_BP(VERIFY_line_62); - // PAPER LINE 62 - rct::key P = rct::addKeys(proof.A, rct::scalarmultKey(proof.S, x)); - PERF_TIMER_STOP(VERIFY_line_62); - - // Compute the number of rounds for the inner product - const size_t rounds = proof.L.size(); - CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + const Bulletproof &proof = *p; + + // check scalar range + CHECK_AND_ASSERT_MES(is_reduced(proof.taux), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.mu), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.a), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.b), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.t), false, "Input scalar not in range"); + + CHECK_AND_ASSERT_MES(proof.V.size() >= 1, false, "V does not have at least one element"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); + CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); + + max_length = std::max(max_length, proof.L.size()); + nV += proof.V.size(); + + // Reconstruct the challenges + PERF_TIMER_START_BP(VERIFY_start); + proof_data.resize(proof_data.size() + 1); + proof_data_t &pd = proof_data.back(); + rct::key hash_cache = rct::hash_to_scalar(proof.V); + pd.y = hash_cache_mash(hash_cache, proof.A, proof.S); + CHECK_AND_ASSERT_MES(!(pd.y == rct::zero()), false, "y == 0"); + pd.z = hash_cache = rct::hash_to_scalar(pd.y); + CHECK_AND_ASSERT_MES(!(pd.z == rct::zero()), false, "z == 0"); + pd.x = hash_cache_mash(hash_cache, pd.z, proof.T1, proof.T2); + CHECK_AND_ASSERT_MES(!(pd.x == rct::zero()), false, "x == 0"); + pd.x_ip = hash_cache_mash(hash_cache, pd.x, proof.taux, proof.mu, proof.t); + CHECK_AND_ASSERT_MES(!(pd.x_ip == rct::zero()), false, "x_ip == 0"); + PERF_TIMER_STOP_BP(VERIFY_start); + + size_t M; + for (pd.logM = 0; (M = 1<<pd.logM) <= maxM && M < proof.V.size(); ++pd.logM); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+pd.logM, false, "Proof is not the expected size"); + + const size_t rounds = pd.logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_21_22); + // PAPER LINES 21-22 + // The inner product challenges are computed per round + pd.w.resize(rounds); + for (size_t i = 0; i < rounds; ++i) + { + pd.w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + CHECK_AND_ASSERT_MES(!(pd.w[i] == rct::zero()), false, "w[i] == 0"); + } + PERF_TIMER_STOP_BP(VERIFY_line_21_22); - PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 - // The inner product challenges are computed per round - rct::keyV w(rounds); - for (size_t i = 0; i < rounds; ++i) - { - w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + pd.inv_offset = inv_offset; + for (size_t i = 0; i < rounds; ++i) + to_invert.push_back(pd.w[i]); + to_invert.push_back(pd.y); + inv_offset += rounds + 1; } - PERF_TIMER_STOP(VERIFY_line_21_22); + CHECK_AND_ASSERT_MES(max_length < 32, false, "At least one proof is too large"); + size_t maxMN = 1u << max_length; - PERF_TIMER_START_BP(VERIFY_line_24_25); - // Basically PAPER LINES 24-25 - // Compute the curvepoints from G[i] and H[i] - rct::key inner_prod = rct::identity(); - rct::key yinvpow = rct::identity(); - rct::key ypow = rct::identity(); + rct::key tmp; - PERF_TIMER_START_BP(VERIFY_line_24_25_invert); - const rct::key yinv = invert(y); - rct::keyV winv(rounds); - for (size_t i = 0; i < rounds; ++i) - winv[i] = invert(w[i]); - PERF_TIMER_STOP(VERIFY_line_24_25_invert); + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(nV + (2 * (10/*logM*/ + logN) + 4) * proofs.size() + 2 * maxMN); + multiexp_data.resize(2 * maxMN); - for (size_t i = 0; i < N; ++i) + PERF_TIMER_START_BP(VERIFY_line_24_25_invert); + const std::vector<rct::key> inverses = invert(to_invert); + PERF_TIMER_STOP_BP(VERIFY_line_24_25_invert); + + // setup weighted aggregates + rct::key z1 = rct::zero(); + rct::key z3 = rct::zero(); + rct::keyV m_z4(maxMN, rct::zero()), m_z5(maxMN, rct::zero()); + rct::key m_y0 = rct::zero(), y1 = rct::zero(); + int proof_data_index = 0; + for (const Bulletproof *p: proofs) { - // Convert the index to binary IN REVERSE and construct the scalar exponent - rct::key g_scalar = proof.a; - rct::key h_scalar; - sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + const Bulletproof &proof = *p; + const proof_data_t &pd = proof_data[proof_data_index++]; + + CHECK_AND_ASSERT_MES(proof.L.size() == 6+pd.logM, false, "Proof is not the expected size"); + const size_t M = 1 << pd.logM; + const size_t MN = M*N; + const rct::key weight_y = rct::skGen(); + const rct::key weight_z = rct::skGen(); + + // pre-multiply some points by 8 + rct::keyV proof8_V = proof.V; for (rct::key &k: proof8_V) k = rct::scalarmult8(k); + rct::keyV proof8_L = proof.L; for (rct::key &k: proof8_L) k = rct::scalarmult8(k); + rct::keyV proof8_R = proof.R; for (rct::key &k: proof8_R) k = rct::scalarmult8(k); + rct::key proof8_T1 = rct::scalarmult8(proof.T1); + rct::key proof8_T2 = rct::scalarmult8(proof.T2); + rct::key proof8_S = rct::scalarmult8(proof.S); + rct::key proof8_A = rct::scalarmult8(proof.A); + + PERF_TIMER_START_BP(VERIFY_line_61); + // PAPER LINE 61 + sc_mulsub(m_y0.bytes, proof.taux.bytes, weight_y.bytes, m_y0.bytes); + + const rct::keyV zpow = vector_powers(pd.z, M+3); + + rct::key k; + const rct::key ip1y = vector_power_sum(pd.y, MN); + sc_mulsub(k.bytes, zpow[2].bytes, ip1y.bytes, rct::zero().bytes); + for (size_t j = 1; j <= M; ++j) + { + CHECK_AND_ASSERT_MES(j+2 < zpow.size(), false, "invalid zpow index"); + sc_mulsub(k.bytes, zpow[j+2].bytes, ip12.bytes, k.bytes); + } + PERF_TIMER_STOP_BP(VERIFY_line_61); - for (size_t j = rounds; j-- > 0; ) + PERF_TIMER_START_BP(VERIFY_line_61rl_new); + sc_muladd(tmp.bytes, pd.z.bytes, ip1y.bytes, k.bytes); + sc_sub(tmp.bytes, proof.t.bytes, tmp.bytes); + sc_muladd(y1.bytes, tmp.bytes, weight_y.bytes, y1.bytes); + for (size_t j = 0; j < proof8_V.size(); j++) + { + sc_mul(tmp.bytes, zpow[j+2].bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_V[j]); + } + sc_mul(tmp.bytes, pd.x.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T1); + rct::key xsq; + sc_mul(xsq.bytes, pd.x.bytes, pd.x.bytes); + sc_mul(tmp.bytes, xsq.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T2); + PERF_TIMER_STOP_BP(VERIFY_line_61rl_new); + + PERF_TIMER_START_BP(VERIFY_line_62); + // PAPER LINE 62 + multiexp_data.emplace_back(weight_z, proof8_A); + sc_mul(tmp.bytes, pd.x.bytes, weight_z.bytes); + multiexp_data.emplace_back(tmp, proof8_S); + PERF_TIMER_STOP_BP(VERIFY_line_62); + + // Compute the number of rounds for the inner product + const size_t rounds = pd.logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_24_25); + // Basically PAPER LINES 24-25 + // Compute the curvepoints from G[i] and H[i] + rct::key yinvpow = rct::identity(); + rct::key ypow = rct::identity(); + + const rct::key *winv = &inverses[pd.inv_offset]; + const rct::key yinv = inverses[pd.inv_offset + rounds]; + + // precalc + PERF_TIMER_START_BP(VERIFY_line_24_25_precalc); + rct::keyV w_cache(1<<rounds); + w_cache[0] = winv[0]; + w_cache[1] = pd.w[0]; + for (size_t j = 1; j < rounds; ++j) { - size_t J = w.size() - j - 1; + const size_t slots = 1<<(j+1); + for (size_t s = slots; s-- > 0; --s) + { + sc_mul(w_cache[s].bytes, w_cache[s/2].bytes, pd.w[j].bytes); + sc_mul(w_cache[s-1].bytes, w_cache[s/2].bytes, winv[j].bytes); + } + } + PERF_TIMER_STOP_BP(VERIFY_line_24_25_precalc); - if ((i & (((size_t)1)<<j)) == 0) + for (size_t i = 0; i < MN; ++i) + { + rct::key g_scalar = proof.a; + rct::key h_scalar; + if (i == 0) + h_scalar = proof.b; + else + sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + + // Convert the index to binary IN REVERSE and construct the scalar exponent + sc_mul(g_scalar.bytes, g_scalar.bytes, w_cache[i].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, w_cache[(~i) & (MN-1)].bytes); + + // Adjust the scalars using the exponents from PAPER LINE 62 + sc_add(g_scalar.bytes, g_scalar.bytes, pd.z.bytes); + CHECK_AND_ASSERT_MES(2+i/N < zpow.size(), false, "invalid zpow index"); + CHECK_AND_ASSERT_MES(i%N < twoN.size(), false, "invalid twoN index"); + sc_mul(tmp.bytes, zpow[2+i/N].bytes, twoN[i%N].bytes); + if (i == 0) { - sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); + sc_add(tmp.bytes, tmp.bytes, pd.z.bytes); + sc_sub(h_scalar.bytes, h_scalar.bytes, tmp.bytes); } else { - sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); + sc_muladd(tmp.bytes, pd.z.bytes, ypow.bytes, tmp.bytes); + sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); } - } - // Adjust the scalars using the exponents from PAPER LINE 62 - sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); - sc_mul(tmp.bytes, zsq.bytes, twoN[i].bytes); - sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); - sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + sc_mulsub(m_z4[i].bytes, g_scalar.bytes, weight_z.bytes, m_z4[i].bytes); + sc_mulsub(m_z5[i].bytes, h_scalar.bytes, weight_z.bytes, m_z5[i].bytes); + + if (i == 0) + { + yinvpow = yinv; + ypow = pd.y; + } + else if (i != MN-1) + { + sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + sc_mul(ypow.bytes, ypow.bytes, pd.y.bytes); + } + } - // Now compute the basepoint's scalar multiplication - // Each of these could be written as a multiexp operation instead - rct::addKeys3(tmp, g_scalar, Gprecomp[i], h_scalar, Hprecomp[i]); - rct::addKeys(inner_prod, inner_prod, tmp); + PERF_TIMER_STOP_BP(VERIFY_line_24_25); - if (i != N-1) + // PAPER LINE 26 + PERF_TIMER_START_BP(VERIFY_line_26_new); + sc_muladd(z1.bytes, proof.mu.bytes, weight_z.bytes, z1.bytes); + for (size_t i = 0; i < rounds; ++i) { - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - sc_mul(ypow.bytes, ypow.bytes, y.bytes); + sc_mul(tmp.bytes, pd.w[i].bytes, pd.w[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, weight_z.bytes); + multiexp_data.emplace_back(tmp, proof8_L[i]); + sc_mul(tmp.bytes, winv[i].bytes, winv[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, weight_z.bytes); + multiexp_data.emplace_back(tmp, proof8_R[i]); } + sc_mulsub(tmp.bytes, proof.a.bytes, proof.b.bytes, proof.t.bytes); + sc_mul(tmp.bytes, tmp.bytes, pd.x_ip.bytes); + sc_muladd(z3.bytes, tmp.bytes, weight_z.bytes, z3.bytes); + PERF_TIMER_STOP_BP(VERIFY_line_26_new); } - PERF_TIMER_STOP(VERIFY_line_24_25); - PERF_TIMER_START_BP(VERIFY_line_26); - // PAPER LINE 26 - rct::key pprime; - sc_sub(tmp.bytes, rct::zero().bytes, proof.mu.bytes); - rct::addKeys(pprime, P, rct::scalarmultBase(tmp)); - - for (size_t i = 0; i < rounds; ++i) + // now check all proofs at once + PERF_TIMER_START_BP(VERIFY_step2_check); + sc_sub(tmp.bytes, m_y0.bytes, z1.bytes); + multiexp_data.emplace_back(tmp, rct::G); + sc_sub(tmp.bytes, z3.bytes, y1.bytes); + multiexp_data.emplace_back(tmp, rct::H); + for (size_t i = 0; i < maxMN; ++i) { - sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); - sc_mul(tmp2.bytes, winv[i].bytes, winv[i].bytes); -#if 1 - ge_dsmp cacheL, cacheR; - rct::precomp(cacheL, proof.L[i]); - rct::precomp(cacheR, proof.R[i]); - rct::addKeys3(tmp, tmp, cacheL, tmp2, cacheR); - rct::addKeys(pprime, pprime, tmp); -#else - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.L[i], tmp)); - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.R[i], tmp2)); -#endif + multiexp_data[i * 2] = {m_z4[i], Gi_p3[i]}; + multiexp_data[i * 2 + 1] = {m_z5[i], Hi_p3[i]}; } - sc_mul(tmp.bytes, proof.t.bytes, x_ip.bytes); - rct::addKeys(pprime, pprime, rct::scalarmultKey(rct::H, tmp)); - PERF_TIMER_STOP(VERIFY_line_26); - - PERF_TIMER_START_BP(VERIFY_step2_check); - sc_mul(tmp.bytes, proof.a.bytes, proof.b.bytes); - sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); - tmp = rct::scalarmultKey(rct::H, tmp); - rct::addKeys(tmp, tmp, inner_prod); - PERF_TIMER_STOP(VERIFY_step2_check); - if (!(pprime == tmp)) - { - MERROR("Verification failure at step 2"); + if (!(multiexp(multiexp_data, 2 * maxMN) == rct::identity())) + { + PERF_TIMER_STOP_BP(VERIFY_step2_check); + MERROR("Verification failure"); return false; } + PERF_TIMER_STOP_BP(VERIFY_step2_check); - PERF_TIMER_STOP(VERIFY); + PERF_TIMER_STOP_BP(VERIFY); return true; } +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs) +{ + std::vector<const Bulletproof*> proof_pointers; + for (const Bulletproof &proof: proofs) + proof_pointers.push_back(&proof); + return bulletproof_VERIFY(proof_pointers); +} + +bool bulletproof_VERIFY(const Bulletproof &proof) +{ + std::vector<const Bulletproof*> proofs; + proofs.push_back(&proof); + return bulletproof_VERIFY(proofs); +} + } diff --git a/src/ringct/bulletproofs.h b/src/ringct/bulletproofs.h index 3061d272e..b86202ccc 100644 --- a/src/ringct/bulletproofs.h +++ b/src/ringct/bulletproofs.h @@ -40,7 +40,11 @@ namespace rct Bulletproof bulletproof_PROVE(const rct::key &v, const rct::key &gamma); Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma); +Bulletproof bulletproof_PROVE(const rct::keyV &v, const rct::keyV &gamma); +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma); bool bulletproof_VERIFY(const Bulletproof &proof); +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs); +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs); } diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc new file mode 100644 index 000000000..6f77fed34 --- /dev/null +++ b/src/ringct/multiexp.cc @@ -0,0 +1,711 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#include "misc_log_ex.h" +#include "common/perf_timer.h" +extern "C" +{ +#include "crypto/crypto-ops.h" +} +#include "common/aligned.h" +#include "rctOps.h" +#include "multiexp.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "multiexp" + +//#define MULTIEXP_PERF(x) x +#define MULTIEXP_PERF(x) + +#define RAW_MEMORY_BLOCK +//#define ALTERNATE_LAYOUT +//#define TRACK_STRAUS_ZERO_IDENTITY + +// per points us for N/B points (B point bands) +// raw alt 128/192 4096/192 4096/4096 +// 0 0 52.6 71 71.2 +// 0 1 53.2 72.2 72.4 +// 1 0 52.7 67 67.1 +// 1 1 52.8 70.4 70.2 + +// Pippenger: +// 1 2 3 4 5 6 7 8 9 bestN +// 2 555 598 621 804 1038 1733 2486 5020 8304 1 +// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 +// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 +// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 +// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 +// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 +// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 +// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 +// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 +// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 +// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 +// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 + +// 2 4 8 16 32 64 128 256 512 1024 2048 4096 +// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 +// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 +// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 +// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 + +// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip +// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip + +// New timings: +// Pippenger: +// 2/1 always +// 3/2 at ~13 +// 4/3 at ~29 +// 5/4 at ~83 +// 6/5 < 200 +// 7/6 at ~470 +// 8/7 at ~1180 +// 9/8 at ~2290 +// Cached Pippenger: +// 6/5 < 200 +// 7/6 at 460 +// 8/7 at 1180 +// 9/8 at 2300 +// +// Cached Straus/Pippenger cross at 232 +// + +namespace rct +{ + +static inline bool operator<(const rct::key &k0, const rct::key&k1) +{ + for (int n = 31; n >= 0; --n) + { + if (k0.bytes[n] < k1.bytes[n]) + return true; + if (k0.bytes[n] > k1.bytes[n]) + return false; + } + return false; +} + +static inline rct::key div2(const rct::key &k) +{ + rct::key res; + int carry = 0; + for (int n = 31; n >= 0; --n) + { + int new_carry = (k.bytes[n] & 1) << 7; + res.bytes[n] = k.bytes[n] / 2 + carry; + carry = new_carry; + } + return res; +} + +static inline rct::key pow2(size_t n) +{ + CHECK_AND_ASSERT_THROW_MES(n < 256, "Invalid pow2 argument"); + rct::key res = rct::zero(); + res[n >> 3] |= 1<<(n&7); + return res; +} + +static inline int test(const rct::key &k, size_t n) +{ + if (n >= 256) return 0; + return k[n >> 3] & (1 << (n & 7)); +} + +static inline void add(ge_p3 &p3, const ge_cached &other) +{ + ge_p1p1 p1; + ge_add(&p1, &p3, &other); + ge_p1p1_to_p3(&p3, &p1); +} + +static inline void add(ge_p3 &p3, const ge_p3 &other) +{ + ge_cached cached; + ge_p3_to_cached(&cached, &other); + add(p3, cached); +} + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 1, "Not enough points"); + std::vector<size_t> heap(points); + for (size_t n = 0; n < points; ++n) + heap[n] = n; + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_cached cached; + ge_p3_to_cached(&cached, &data[index1].point); + ge_p1p1 p1; + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 0, "Not enough points"); + std::vector<size_t> heap; + heap.reserve(points); + for (size_t n = 0; n < points; ++n) + { + if (!(data[n].scalar == rct::zero()) && !ge_p3_is_point_at_infinity(&data[n].point)) + heap.push_back(n); + } + points = heap.size(); + if (points == 0) + return rct::identity(); + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + + if (points < 2) + { + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(div, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + ge_cached cached; + ge_p1p1 p1; + ge_p2 p2; + + MULTIEXP_PERF(PERF_TIMER_RESUME(div)); + while (1) + { + rct::key s1_2 = div2(data[index1].scalar); + if (!(data[index2].scalar < s1_2)) + break; + if (data[index1].scalar.bytes[0] & 1) + { + data.resize(data.size()+1); + data.back().scalar = rct::identity(); + data.back().point = data[index1].point; + heap.push_back(data.size() - 1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + data[index1].scalar = div2(data[index1].scalar); + ge_p3_to_p2(&p2, &data[index1].point); + ge_p2_dbl(&p1, &p2); + ge_p1p1_to_p3(&data[index1].point, &p1); + } + MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_p3_to_cached(&cached, &data[index1].point); + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +#define STRAUS_C 4 + +struct straus_cached_data +{ +#ifdef RAW_MEMORY_BLOCK + size_t size; + ge_cached *multiples; + straus_cached_data(): size(0), multiples(NULL) {} + ~straus_cached_data() { aligned_free(multiples); } +#else + std::vector<std::vector<ge_cached>> multiples; +#endif +}; +#ifdef RAW_MEMORY_BLOCK +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)*((1<<STRAUS_C)-1)+((digit)-1)] +#else +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)+cache->size*((digit)-1)] +#endif +#else +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[j][digit-1] +#else +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[digit][j] +#endif +#endif + +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(multiples, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + std::shared_ptr<straus_cached_data> cache(new straus_cached_data()); + +#ifdef RAW_MEMORY_BLOCK + const size_t offset = cache->size; + cache->multiples = (ge_cached*)aligned_realloc(cache->multiples, sizeof(ge_cached) * ((1<<STRAUS_C)-1) * std::max(offset, N), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->multiples, "Out of memory"); + cache->size = N; + for (size_t j=offset;j<N;++j) + { + ge_p3_to_cached(&CACHE_OFFSET(cache, j, 1), &data[j].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &CACHE_OFFSET(cache, j, i-1)); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&CACHE_OFFSET(cache, j, i), &p3); + } + } +#else +#ifdef ALTERNATE_LAYOUT + const size_t offset = cache->multiples.size(); + cache->multiples.resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + { + cache->multiples[i].resize((1<<STRAUS_C)-1); + ge_p3_to_cached(&cache->multiples[i][0], &data[i].point); + for (size_t j=2;j<1<<STRAUS_C;++j) + { + ge_add(&p1, &data[i].point, &cache->multiples[i][j-2]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j-1], &p3); + } + } +#else + cache->multiples.resize(1<<STRAUS_C); + size_t offset = cache->multiples[1].size(); + cache->multiples[1].resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + ge_p3_to_cached(&cache->multiples[1][i], &data[i].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + cache->multiples[i].resize(std::max(offset, N)); + for (size_t j=offset;j<N;++j) + { + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &cache->multiples[i-1][j]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j], &p3); + } + } +#endif +#endif + MULTIEXP_PERF(PERF_TIMER_STOP(multiples)); + + return cache; +} + +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache) +{ + size_t sz = 0; +#ifdef RAW_MEMORY_BLOCK + sz += cache->size * sizeof(ge_cached) * ((1<<STRAUS_C)-1); +#else + for (const auto &e0: cache->multiples) + sz += e0.size() * sizeof(ge_cached); +#endif + return sz; +} + +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache, size_t STEP) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); + bool HiGi = cache != NULL; + STEP = STEP ? STEP : 192; + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + static constexpr unsigned int mask = (1<<STRAUS_C)-1; + std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + +#ifdef TRACK_STRAUS_ZERO_IDENTITY + MULTIEXP_PERF(PERF_TIMER_START_UNIT(skip, 1000000)); + std::vector<uint8_t> skip(data.size()); + for (size_t i = 0; i < data.size(); ++i) + skip[i] = data[i].scalar == rct::zero() || ge_p3_is_point_at_infinity(&data[i].point); + MULTIEXP_PERF(PERF_TIMER_STOP(skip)); +#endif + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(digits, 1000000)); +#if STRAUS_C==4 + std::unique_ptr<uint8_t[]> digits{new uint8_t[64 * data.size()]}; +#else + std::unique_ptr<uint8_t[]> digits{new uint8_t[256 * data.size()]}; +#endif + for (size_t j = 0; j < data.size(); ++j) + { + const unsigned char *bytes = data[j].scalar.bytes; +#if STRAUS_C==4 + unsigned int i; + for (i = 0; i < 64; i += 2, bytes++) + { + digits[j*64+i] = bytes[0] & 0xf; + digits[j*64+i+1] = bytes[0] >> 4; + } +#elif 1 + unsigned char bytes33[33]; + memcpy(bytes33, data[j].scalar.bytes, 32); + bytes33[32] = 0; + bytes = bytes33; + for (size_t i = 0; i < 256; ++i) + digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; +#else + rct::key shifted = data[j].scalar; + for (size_t i = 0; i < 256; ++i) + { + digits[j*256+i] = shifted.bytes[0] & 0xf; + shifted = div2(shifted, (256-i)>>3); + } +#endif + } + MULTIEXP_PERF(PERF_TIMER_STOP(digits)); + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + size_t start_i = 0; + while (start_i < 256 && !(maxscalar < pow2(start_i))) + start_i += STRAUS_C; + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + ge_p3 res_p3 = ge_p3_identity; + + for (size_t start_offset = 0; start_offset < data.size(); start_offset += STEP) + { + const size_t num_points = std::min(data.size() - start_offset, STEP); + + ge_p3 band_p3 = ge_p3_identity; + size_t i = start_i; + if (!(i < STRAUS_C)) + goto skipfirst; + while (!(i < STRAUS_C)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &band_p3); + for (size_t j = 0; j < STRAUS_C; ++j) + { + ge_p2_dbl(&p1, &p2); + if (j == STRAUS_C - 1) + ge_p1p1_to_p3(&band_p3, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } +skipfirst: + i -= STRAUS_C; + for (size_t j = start_offset; j < start_offset + num_points; ++j) + { +#ifdef TRACK_STRAUS_ZERO_IDENTITY + if (skip[j]) + continue; +#endif +#if STRAUS_C==4 + const uint8_t digit = digits[j*64+i/4]; +#else + const uint8_t digit = digits[j*256+i]; +#endif + if (digit) + { + ge_add(&p1, &band_p3, &CACHE_OFFSET(local_cache, j, digit)); + ge_p1p1_to_p3(&band_p3, &p1); + } + } + } + + ge_p3_to_cached(&cached, &band_p3); + ge_add(&p1, &res_p3, &cached); + ge_p1p1_to_p3(&res_p3, &p1); + } + + rct::key res; + ge_p3_tobytes(res.bytes, &res_p3); + return res; +} + +size_t get_pippenger_c(size_t N) +{ + if (N <= 13) return 2; + if (N <= 29) return 3; + if (N <= 83) return 4; + if (N <= 185) return 5; + if (N <= 465) return 6; + if (N <= 1180) return 7; + if (N <= 2295) return 8; + return 9; +} + +struct pippenger_cached_data +{ + size_t size; + ge_cached *cached; + pippenger_cached_data(): size(0), cached(NULL) {} + ~pippenger_cached_data() { aligned_free(cached); } +}; + +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); + CHECK_AND_ASSERT_THROW_MES(start_offset <= data.size(), "Bad cache base data"); + if (N == 0) + N = data.size() - start_offset; + CHECK_AND_ASSERT_THROW_MES(N <= data.size() - start_offset, "Bad cache base data"); + ge_cached cached; + std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); + + cache->size = N; + cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); + for (size_t i = 0; i < N; ++i) + ge_p3_to_cached(&cache->cached[i], &data[i+start_offset].point); + + MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); + return cache; +} + +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache) +{ + return cache->size * sizeof(*cache->cached); +} + +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t cache_size, size_t c) +{ + if (cache != NULL && cache_size == 0) + cache_size = cache->size; + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size, "Cache is too small"); + if (c == 0) + c = get_pippenger_c(data.size()); + CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); + + ge_p3 result = ge_p3_identity; + bool result_init = false; + std::unique_ptr<ge_p3[]> buckets{new ge_p3[1<<c]}; + bool buckets_init[1<<9]; + std::shared_ptr<pippenger_cached_data> local_cache = cache == NULL ? pippenger_init_cache(data) : cache; + std::shared_ptr<pippenger_cached_data> local_cache_2 = data.size() > cache_size ? pippenger_init_cache(data, cache_size) : NULL; + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + { + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + } + size_t groups = 0; + while (groups < 256 && !(maxscalar < pow2(groups))) + ++groups; + groups = (groups + c - 1) / c; + + for (size_t k = groups; k-- > 0; ) + { + if (result_init) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &result); + for (size_t i = 0; i < c; ++i) + { + ge_p1p1 p1; + ge_p2_dbl(&p1, &p2); + if (i == c - 1) + ge_p1p1_to_p3(&result, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } + } + memset(buckets_init, 0, 1u<<c); + + // partition scalars into buckets + for (size_t i = 0; i < data.size(); ++i) + { + unsigned int bucket = 0; + for (size_t j = 0; j < c; ++j) + if (test(data[i].scalar, k*c+j)) + bucket |= 1<<j; + if (bucket == 0) + continue; + CHECK_AND_ASSERT_THROW_MES(bucket < (1u<<c), "bucket overflow"); + if (buckets_init[bucket]) + { + if (i < cache_size) + add(buckets[bucket], local_cache->cached[i]); + else + add(buckets[bucket], local_cache_2->cached[i - cache_size]); + } + else + { + buckets[bucket] = data[i].point; + buckets_init[bucket] = true; + } + } + + // sum the buckets + ge_p3 pail; + bool pail_init = false; + for (size_t i = (1<<c)-1; i > 0; --i) + { + if (buckets_init[i]) + { + if (pail_init) + add(pail, buckets[i]); + else + { + pail = buckets[i]; + pail_init = true; + } + } + if (pail_init) + { + if (result_init) + add(result, pail); + else + { + result = pail; + result_init = true; + } + } + } + } + + rct::key res; + ge_p3_tobytes(res.bytes, &result); + return res; +} + +} diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h new file mode 100644 index 000000000..b52707933 --- /dev/null +++ b/src/ringct/multiexp.h @@ -0,0 +1,71 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#pragma once + +#ifndef MULTIEXP_H +#define MULTIEXP_H + +#include <vector> +#include "crypto/crypto.h" +#include "rctTypes.h" +#include "misc_log_ex.h" + +namespace rct +{ + +struct MultiexpData { + rct::key scalar; + ge_p3 point; + + MultiexpData() {} + MultiexpData(const rct::key &s, const ge_p3 &p): scalar(s), point(p) {} + MultiexpData(const rct::key &s, const rct::key &p): scalar(s) + { + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&point, p.bytes) == 0, "ge_frombytes_vartime failed"); + } +}; + +struct straus_cached_data; +struct pippenger_cached_data; + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data); +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data); +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache); +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset = 0, size_t N =0); +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache); +size_t get_pippenger_c(size_t N); +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t cache_size = 0, size_t c = 0); + +} + +#endif diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index cc46d0aa7..41bbf6ca3 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -60,16 +60,26 @@ namespace rct { //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data) + { + if (ge_frombytes_vartime(P, data)) + return false; + ge_p2 R; + ge_scalarmult(&R, curveOrder().bytes, P); + key tmp; + ge_tobytes(tmp.bytes, &R); + return tmp == identity(); + } + //generates a random scalar which can be used as a secret key or mask void skGen(key &sk) { - sk = crypto::rand<key>(); - sc_reduce32(sk.bytes); + random32_unbiased(sk.bytes); } //generates a random scalar which can be used as a secret key or mask key skGen() { - key sk = crypto::rand<key>(); - sc_reduce32(sk.bytes); + key sk; + skGen(sk); return sk; } @@ -79,9 +89,8 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES(rows > 0, "0 keys requested"); keyV rv(rows); size_t i = 0; - crypto::rand(rows * sizeof(key), (uint8_t*)&rv[0]); for (i = 0 ; i < rows ; i++) { - sc_reduce32(rv[i].bytes); + skGen(rv[i]); } return rv; } @@ -134,12 +143,9 @@ namespace rct { } key zeroCommit(xmr_amount amount) { - key mask = identity(); - mask = scalarmultBase(mask); key am = d2h(amount); key bH = scalarmultH(am); - addKeys(mask, mask, bH); - return mask; + return addKeys(G, bH); } key commit(xmr_amount amount, const key &mask) { @@ -198,15 +204,33 @@ namespace rct { //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a) { - ge_p3 A; ge_p2 R; - CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, H.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); - ge_scalarmult(&R, a.bytes, &A); + ge_scalarmult(&R, a.bytes, &ge_p3_H); key aP; ge_tobytes(aP.bytes, &R); return aP; } + //Computes 8P + key scalarmult8(const key & P) { + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); + ge_p2 p2; + ge_p3_to_p2(&p2, &p3); + ge_p1p1 p1; + ge_mul8(&p1, &p2); + ge_p1p1_to_p2(&p2, &p1); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + //Computes aL where L is the curve order + bool isInMainSubgroup(const key & a) { + ge_p3 p3; + return toPointCheckOrder(&p3, a.bytes); + } + //Curve addition / subtractions //for curve points: AB = A + B @@ -228,6 +252,25 @@ namespace rct { return k; } + rct::key addKeys(const keyV &A) { + if (A.empty()) + return rct::identity(); + ge_p3 p3, tmp; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, A[0].bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); + for (size_t i = 1; i < A.size(); ++i) + { + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&tmp, A[i].bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); + ge_cached p2; + ge_p3_to_cached(&p2, &tmp); + ge_p1p1 p1; + ge_add(&p1, &p3, &p2); + ge_p1p1_to_p3(&p3, &p1); + } + rct::key res; + ge_p3_tobytes(res.bytes, &p3); + return res; + } + //addKeys1 //aGB = aG + B where a is a scalar, G is the basepoint, and B is a point void addKeys1(key &aGB, const key &a, const key & B) { diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 3f8f6955c..60e920b3a 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -62,6 +62,9 @@ namespace rct { static const key Z = { {0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key I = { {0x01, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key L = { {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; + static const key G = { {0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } }; + static const key EIGHT = { {0x08, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; + static const key INV_EIGHT = { { 0x79, 0x2f, 0xdc, 0xe2, 0x29, 0xe5, 0x06, 0x61, 0xd0, 0xda, 0x1c, 0x7d, 0xb3, 0x9d, 0xd3, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06 } }; //Creates a zero scalar inline key zero() { return Z; } @@ -82,6 +85,7 @@ namespace rct { keyM keyMInit(size_t rows, size_t cols); //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data); //generates a random scalar which can be used as a secret key or mask key skGen(); @@ -118,12 +122,17 @@ namespace rct { key scalarmultKey(const key &P, const key &a); //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a); + // multiplies a point by 8 + key scalarmult8(const key & P); + // checks a is in the main subgroup (ie, not a small one) + bool isInMainSubgroup(const key & a); //Curve addition / subtractions //for curve points: AB = A + B void addKeys(key &AB, const key &A, const key &B); rct::key addKeys(const key &A, const key &B); + rct::key addKeys(const keyV &A); //aGB = aG + B where a is a scalar, G is the basepoint, and B is a point void addKeys1(key &aGB, const key &a, const key & B); //aGbB = aG + bB where a, b are scalars, G is the basepoint and B is a point diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 777b4d13a..dccd18867 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -44,37 +44,26 @@ using namespace std; #define CHECK_AND_ASSERT_MES_L1(expr, ret, message) {if(!(expr)) {MCERROR("verify", message); return ret;}} -namespace rct { - bool is_simple(int type) - { - switch (type) - { - case RCTTypeSimple: - case RCTTypeSimpleBulletproof: - return true; - default: - return false; - } - } - - bool is_bulletproof(int type) +namespace +{ + rct::Bulletproof make_dummy_bulletproof(size_t n_outs) { - switch (type) - { - case RCTTypeSimpleBulletproof: - case RCTTypeFullBulletproof: - return true; - default: - return false; - } + const rct::key I = rct::identity(); + size_t nrl = 0; + while ((1u << nrl) < n_outs) + ++nrl; + nrl += 6; + return rct::Bulletproof{rct::keyV(n_outs, I), I, I, I, I, I, I, rct::keyV(nrl, I), rct::keyV(nrl, I), I, I, I}; } +} - Bulletproof proveRangeBulletproof(key &C, key &mask, uint64_t amount) +namespace rct { + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) { - mask = rct::skGen(); - Bulletproof proof = bulletproof_PROVE(amount, mask); - CHECK_AND_ASSERT_THROW_MES(proof.V.size() == 1, "V has not exactly one element"); - C = proof.V[0]; + masks = rct::skvGen(amounts.size()); + Bulletproof proof = bulletproof_PROVE(amounts, masks); + CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); + C = proof.V; return proof; } @@ -85,6 +74,13 @@ namespace rct { catch (...) { return false; } } + bool verBulletproof(const std::vector<const Bulletproof*> &proofs) + { + try { return bulletproof_VERIFY(proofs); } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (...) { return false; } + } + //Borromean (c.f. gmax/andytoshi's paper) boroSig genBorromean(const key64 x, const key64 P1, const key64 P2, const bits indices) { key64 L[2], alpha; @@ -147,7 +143,7 @@ namespace rct { //This is a just slghtly more efficient version than the ones described below //(will be explained in more detail in Ring Multisig paper //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -244,7 +240,7 @@ namespace rct { //This is a just slghtly more efficient version than the ones described below //(will be explained in more detail in Ring Multisig paper //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -285,6 +281,7 @@ namespace rct { for (j = 0; j < dsRows; j++) { addKeys2(L, rv.ss[i][j], c_old, pk[i][j]); hashToPoint(Hi, pk[i][j]); + CHECK_AND_ASSERT_MES(!(Hi == rct::identity()), false, "Data hashed to point at infinity"); addKeys3(R, rv.ss[i][j], Hi, c_old, Ip[j].k); toHash[3 * j + 1] = pk[i][j]; toHash[3 * j + 2] = L; @@ -307,7 +304,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -339,7 +336,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -389,7 +386,7 @@ namespace rct { std::stringstream ss; binary_archive<true> ba(ss); CHECK_AND_ASSERT_THROW_MES(!rv.mixRing.empty(), "Empty mixRing"); - const size_t inputs = is_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); + const size_t inputs = is_rct_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); const size_t outputs = rv.ecdhInfo.size(); key prehash; CHECK_AND_ASSERT_THROW_MES(const_cast<rctSig&>(rv).serialize_rctsig_base(ba, inputs, outputs), @@ -398,7 +395,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeSimpleBulletproof || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeBulletproof) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -441,7 +438,7 @@ namespace rct { //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs @@ -492,7 +489,9 @@ namespace rct { for (size_t j = 0; j < outPk.size(); j++) { sc_sub(sk[rows].bytes, sk[rows].bytes, outSk[j].mask.bytes); //subtract output masks in last row.. } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(sk.data(), sk.size() * sizeof(key)); + return result; } @@ -521,13 +520,15 @@ namespace rct { M[i][0] = pubs[i].dest; subKeys(M[i][1], pubs[i].mask, Cout); } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(&sk[0], sizeof(key)); + return result; } //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs @@ -650,12 +651,12 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -665,13 +666,10 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES((kLRki && msout) || (!kLRki && !msout), "Only one of kLRki/msout is present"); rctSig rv; - rv.type = bulletproof ? RCTTypeFullBulletproof : RCTTypeFull; + rv.type = RCTTypeFull; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else - rv.p.rangeSigs.resize(destinations.size()); + rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i = 0; @@ -681,17 +679,10 @@ namespace rct { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, amounts[i]); - else - rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); + rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else - CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); + CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); @@ -721,12 +712,13 @@ namespace rct { ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev) { + const bool bulletproof = range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -742,35 +734,94 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeSimpleBulletproof : RCTTypeSimple; + rv.type = bulletproof ? RCTTypeBulletproof : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else + if (!bulletproof) rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i; keyV masks(destinations.size()); //sk mask.. outSk.resize(destinations.size()); - key sumout = zero(); for (i = 0; i < destinations.size(); i++) { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); - else + if (!bulletproof) rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else + if (!bulletproof) CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - + } + + rv.p.bulletproofs.clear(); + if (bulletproof) + { + std::vector<uint64_t> proof_amounts; + size_t n_amounts = outamounts.size(); + size_t amounts_proved = 0; + if (range_proof_type == RangeProofPaddedBulletproof) + { + rct::keyV C, masks; + if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) + { + // use a fake bulletproof for speed + rv.p.bulletproofs.push_back(make_dummy_bulletproof(outamounts.size())); + C = rct::keyV(outamounts.size(), I); + masks = rct::keyV(outamounts.size(), I); + } + else + { + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + } + for (i = 0; i < outamounts.size(); ++i) + { + rv.outPk[i].mask = rct::scalarmult8(C[i]); + outSk[i].mask = masks[i]; + } + } + else while (amounts_proved < n_amounts) + { + size_t batch_size = 1; + if (range_proof_type == RangeProofMultiOutputBulletproof) + while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) + batch_size *= 2; + rct::keyV C, masks; + std::vector<uint64_t> batch_amounts(batch_size); + for (i = 0; i < batch_size; ++i) + batch_amounts[i] = outamounts[i + amounts_proved]; + if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) + { + // use a fake bulletproof for speed + rv.p.bulletproofs.push_back(make_dummy_bulletproof(batch_amounts.size())); + C = rct::keyV(batch_amounts.size(), I); + masks = rct::keyV(batch_amounts.size(), I); + } + else + { + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + } + for (i = 0; i < batch_size; ++i) + { + rv.outPk[i + amounts_proved].mask = rct::scalarmult8(C[i]); + outSk[i + amounts_proved].mask = masks[i]; + } + amounts_proved += batch_size; + } + } + + key sumout = zero(); + for (i = 0; i < outSk.size(); ++i) + { sc_add(sumout.bytes, outSk[i].mask.bytes, sumout.bytes); //mask amount and mask @@ -818,7 +869,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, RangeProofBorromean, hwdev); } //RingCT protocol @@ -828,18 +879,15 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number bool verRct(const rctSig & rv, bool semantics) { PERF_TIMER(verRct); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "verRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "verRct called on non-full rctSig"); if (semantics) { - if (rv.type == RCTTypeFullBulletproof) - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); - else - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "full rctSig has not one MG"); } @@ -856,19 +904,13 @@ namespace rct { tools::threadpool::waiter waiter; std::deque<bool> results(rv.outPk.size(), false); DP("range proofs verified?"); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); - } - waiter.wait(); + for (size_t i = 0; i < rv.outPk.size(); i++) + tpool.submit(&waiter, [&, i] { results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); + waiter.wait(&tpool); - for (size_t i = 0; i < rv.outPk.size(); ++i) { + for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); + LOG_PRINT_L1("Range proof verified failed for proof " << i); return false; } } @@ -902,17 +944,26 @@ namespace rct { //ver RingCT simple //assumes only post-rct style inputs (at least for max anonymity) - bool verRctSimple(const rctSig & rv, bool semantics) { + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rvv) { try { - PERF_TIMER(verRctSimple); + PERF_TIMER(verRctSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "verRctSimple called on non simple rctSig"); - if (semantics) + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + std::deque<bool> results; + std::vector<const Bulletproof*> proofs; + size_t max_non_bp_proofs = 0, offset = 0; + + for (const rctSig *rvp: rvv) { - if (rv.type == RCTTypeSimpleBulletproof) + CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); + const rctSig &rv = *rvp; + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + if (bulletproof) { - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == n_bulletproof_amounts(rv.p.bulletproofs), false, "Mismatched sizes of outPk and bulletproofs"); CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.p.MGs.size(), false, "Mismatched sizes of rv.p.pseudoOuts and rv.p.MGs"); CHECK_AND_ASSERT_MES(rv.pseudoOuts.empty(), false, "rv.pseudoOuts is not empty"); } @@ -923,81 +974,121 @@ namespace rct { CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.empty(), false, "rv.p.pseudoOuts is not empty"); } CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); - } - else - { - // semantics check is early, and mixRing/MGs aren't resolved yet - if (rv.type == RCTTypeSimpleBulletproof) - CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); - else - CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); - } - const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + if (!bulletproof) + max_non_bp_proofs += rv.p.rangeSigs.size(); + } - std::deque<bool> results(threads); - tools::threadpool& tpool = tools::threadpool::getInstance(); - tools::threadpool::waiter waiter; + results.resize(max_non_bp_proofs); + for (const rctSig *rvp: rvv) + { + const rctSig &rv = *rvp; - const keyV &pseudoOuts = is_bulletproof(rv.type) ? rv.p.pseudoOuts : rv.pseudoOuts; + const bool bulletproof = is_rct_bulletproof(rv.type); + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; - if (semantics) { - key sumOutpks = identity(); + rct::keyV masks(rv.outPk.size()); for (size_t i = 0; i < rv.outPk.size(); i++) { - addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); + masks[i] = rv.outPk[i].mask; } + key sumOutpks = addKeys(masks); DP(sumOutpks); - key txnFeeKey = scalarmultH(d2h(rv.txnFee)); + const key txnFeeKey = scalarmultH(d2h(rv.txnFee)); addKeys(sumOutpks, txnFeeKey, sumOutpks); - key sumPseudoOuts = identity(); - for (size_t i = 0 ; i < pseudoOuts.size() ; i++) { - addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); - } + key sumPseudoOuts = addKeys(pseudoOuts); DP(sumPseudoOuts); //check pseudoOuts vs Outs.. if (!equalKeys(sumPseudoOuts, sumOutpks)) { - LOG_PRINT_L1("Sum check failed"); - return false; + LOG_PRINT_L1("Sum check failed"); + return false; } - results.clear(); - results.resize(rv.outPk.size()); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); + if (bulletproof) + { + for (size_t i = 0; i < rv.p.bulletproofs.size(); i++) + proofs.push_back(&rv.p.bulletproofs[i]); } - waiter.wait(); - - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); - return false; - } + else + { + for (size_t i = 0; i < rv.p.rangeSigs.size(); i++) + tpool.submit(&waiter, [&, i, offset] { results[i+offset] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); + offset += rv.p.rangeSigs.size(); } } - else { - const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); - - results.clear(); - results.resize(rv.mixRing.size()); - for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { - tpool.submit(&waiter, [&, i] { - results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); - }); + if (!proofs.empty() && !verBulletproof(proofs)) + { + LOG_PRINT_L1("Aggregate range proof verified failed"); + return false; + } + + waiter.wait(&tpool); + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("Range proof verified failed for proof " << i); + return false; } - waiter.wait(); + } - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("verRctMGSimple failed for input " << i); - return false; - } + return true; + } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (const std::exception &e) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple: " << e.what()); + return false; + } + catch (...) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple, but not an actual exception"); + return false; + } + } + + bool verRctSemanticsSimple(const rctSig & rv) + { + return verRctSemanticsSimple(std::vector<const rctSig*>(1, &rv)); + } + + //ver RingCT simple + //assumes only post-rct style inputs (at least for max anonymity) + bool verRctNonSemanticsSimple(const rctSig & rv) { + try + { + PERF_TIMER(verRctNonSemanticsSimple); + + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctNonSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + // semantics check is early, and mixRing/MGs aren't resolved yet + if (bulletproof) + CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); + else + CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); + + const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + + std::deque<bool> results(threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; + + const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); + + results.clear(); + results.resize(rv.mixRing.size()); + for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { + tpool.submit(&waiter, [&, i] { + results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); + }); + } + waiter.wait(&tpool); + + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("verRctMGSimple failed for input " << i); + return false; } } @@ -1006,12 +1097,12 @@ namespace rct { // we can get deep throws from ge_frombytes_vartime if input isn't valid catch (const std::exception &e) { - LOG_PRINT_L1("Error in verRct: " << e.what()); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple: " << e.what()); return false; } catch (...) { - LOG_PRINT_L1("Error in verRct, but not an actual exception"); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple, but not an actual exception"); return false; } } @@ -1023,11 +1114,11 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "decodeRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "decodeRct called on non-full rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1040,6 +1131,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1055,7 +1148,7 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1068,6 +1161,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1083,12 +1178,12 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeFullBulletproof || rv.type == RCTTypeSimpleBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); CHECK_AND_ASSERT_MES(k.size() == msout.c.size(), false, "Mismatched k/msout.c size"); - if (rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeFull) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "MGs not a single element"); } diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index b8aab0f11..b67a0b992 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -70,7 +70,7 @@ namespace rct { //Multilayered Spontaneous Anonymous Group Signatures (MLSAG signatures) //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -80,7 +80,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -90,7 +90,7 @@ namespace rct { //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs @@ -116,22 +116,24 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev); rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } - bool verRctSimple(const rctSig & rv, bool semantics); - static inline bool verRctSimple(const rctSig & rv) { return verRctSimple(rv, true) && verRctSimple(rv, false); } + bool verRctSemanticsSimple(const rctSig & rv); + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rv); + bool verRctNonSemanticsSimple(const rctSig & rv); + static inline bool verRctSimple(const rctSig & rv) { return verRctSemanticsSimple(rv) && verRctNonSemanticsSimple(rv); } xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); - + key get_pre_mlsag_hash(const rctSig &rv, hw::device &hwdev); bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key); } #endif /* RCTSIGS_H */ diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 5650b3ba1..90ed65df0 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -28,6 +28,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "misc_log_ex.h" +#include "cryptonote_config.h" #include "rctTypes.h" using namespace crypto; using namespace std; @@ -209,4 +211,90 @@ namespace rct { return vali; } + bool is_rct_simple(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_bulletproof(int type) + { + switch (type) + { + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_borromean(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeFull: + return true; + default: + return false; + } + } + + size_t n_bulletproof_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.V.size() <= (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() * 2 > (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() > 0, 0, "Empty bulletproof"); + return proof.V.size(); + } + + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + + size_t n_bulletproof_max_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + return 1 << (proof.L.size() - 6); + } + + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_max_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index eba1e3d93..18290637b 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -36,6 +36,7 @@ #include <vector> #include <iostream> #include <cinttypes> +#include <sodium/crypto_verify_32.h> extern "C" { #include "crypto/crypto-ops.h" @@ -81,7 +82,7 @@ namespace rct { unsigned char operator[](int i) const { return bytes[i]; } - bool operator==(const key &k) const { return !memcmp(bytes, k.bytes, sizeof(bytes)); } + bool operator==(const key &k) const { return !crypto_verify_32(bytes, k.bytes); } unsigned char bytes[32]; }; typedef std::vector<key> keyV; //vector of keys @@ -150,7 +151,7 @@ namespace rct { }; //just contains the necessary keys to represent MLSAG sigs - //c.f. http://eprint.iacr.org/2015/1098 + //c.f. https://eprint.iacr.org/2015/1098 struct mgSig { keyM ss; key cc; @@ -189,6 +190,8 @@ namespace rct { Bulletproof() {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} + Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): + V(V), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} BEGIN_SERIALIZE_OBJECT() // Commitments aren't saved, they're restored via outPk @@ -210,6 +213,11 @@ namespace rct { END_SERIALIZE() }; + size_t n_bulletproof_amounts(const Bulletproof &proof); + size_t n_bulletproof_max_amounts(const Bulletproof &proof); + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs); + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs); + //A container to hold all signatures necessary for RingCT // rangeSigs holds all the rangeproof data of a transaction // MG holds the MLSAG signature of a transaction @@ -221,9 +229,9 @@ namespace rct { RCTTypeNull = 0, RCTTypeFull = 1, RCTTypeSimple = 2, - RCTTypeFullBulletproof = 3, - RCTTypeSimpleBulletproof = 4, + RCTTypeBulletproof = 3, }; + enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; struct rctSigBase { uint8_t type; key message; @@ -240,7 +248,7 @@ namespace rct { FIELD(type) if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -301,21 +309,25 @@ namespace rct { { if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; - if (type == RCTTypeSimpleBulletproof || type == RCTTypeFullBulletproof) + if (type == RCTTypeBulletproof) { + uint32_t nbp = bulletproofs.size(); + FIELD(nbp) ar.tag("bp"); ar.begin_array(); - PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, bulletproofs); - if (bulletproofs.size() != outputs) + if (nbp > outputs) return false; - for (size_t i = 0; i < outputs; ++i) + PREPARE_CUSTOM_VECTOR_SERIALIZATION(nbp, bulletproofs); + for (size_t i = 0; i < nbp; ++i) { FIELDS(bulletproofs[i]) - if (outputs - i > 1) + if (nbp - i > 1) ar.delimit_array(); } + if (n_bulletproof_max_amounts(bulletproofs) < outputs) + return false; ar.end_array(); } else @@ -338,7 +350,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -356,7 +368,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -382,7 +394,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeSimpleBulletproof) + if (type == RCTTypeBulletproof) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -403,6 +415,16 @@ namespace rct { }; struct rctSig: public rctSigBase { rctSigPrunable p; + + keyV& get_pseudo_outs() + { + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + } + + keyV const& get_pseudo_outs() const + { + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + } }; //other basepoint H = toPoint(cn_fast_hash(G)), G the basepoint @@ -506,24 +528,28 @@ namespace rct { //int[64] to uint long long xmr_amount b2d(bits amountb); - static inline const rct::key pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } - static inline const rct::key sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } - static inline const rct::key ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } - static inline const rct::key hash2rct(const crypto::hash &h) { return (const rct::key&)h; } - static inline const crypto::public_key rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } - static inline const crypto::secret_key rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } - static inline const crypto::key_image rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } - static inline const crypto::hash rct2hash(const rct::key &k) { return (const crypto::hash&)k; } - static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return memcmp(&k0, &k1, 32); } + bool is_rct_simple(int type); + bool is_rct_bulletproof(int type); + bool is_rct_borromean(int type); + + static inline const rct::key &pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } + static inline const rct::key &sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } + static inline const rct::key &ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } + static inline const rct::key &hash2rct(const crypto::hash &h) { return (const rct::key&)h; } + static inline const crypto::public_key &rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } + static inline const crypto::secret_key &rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } + static inline const crypto::key_image &rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } + static inline const crypto::hash &rct2hash(const rct::key &k) { return (const crypto::hash&)k; } + static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } + static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } } namespace cryptonote { - static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } - static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } + static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } } namespace rct { diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 7162317ed..8fc42b7e3 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -31,6 +31,7 @@ set(rpc_base_sources set(rpc_sources core_rpc_server.cpp + rpc_handler.cpp instanciations) set(daemon_messages_sources @@ -45,7 +46,8 @@ set(daemon_rpc_server_sources set(rpc_base_headers rpc_args.h) -set(rpc_headers) +set(rpc_headers + rpc_handler.cpp) set(daemon_rpc_server_headers) @@ -63,7 +65,6 @@ set(daemon_rpc_server_private_headers message.h daemon_messages.h daemon_handler.h - rpc_handler.h zmq_server.h) @@ -111,7 +112,6 @@ target_link_libraries(rpc common cryptonote_core cryptonote_protocol - epee ${Boost_REGEX_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index dc7b6b30f..574f6c126 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -45,9 +45,9 @@ using namespace epee; #include "storages/http_abstract_invoke.h" #include "crypto/hash.h" #include "rpc/rpc_args.h" +#include "rpc/rpc_handler.h" #include "core_rpc_server_error_codes.h" #include "p2p/net_node.h" -#include "get_output_distribution_cache.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -194,11 +194,12 @@ namespace cryptonote res.mainnet = m_nettype == MAINNET; res.testnet = m_nettype == TESTNET; res.stagenet = m_nettype == STAGENET; + res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; - res.start_time = (uint64_t)m_core.get_start_time(); + res.start_time = m_restricted ? 0 : (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); res.offline = m_core.offline(); res.bootstrap_daemon_address = m_bootstrap_daemon_address; @@ -207,17 +208,20 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); + res.update_available = m_core.is_update_available(); return true; } //------------------------------------------------------------------------------------------------------------------------------ - static cryptonote::blobdata get_pruned_tx_blob(cryptonote::transaction &tx) - { - std::stringstream ss; - binary_archive<true> ba(ss); - bool r = tx.serialize_base(ba); - CHECK_AND_ASSERT_MES(r, cryptonote::blobdata(), "Failed to serialize rct signatures base"); - return ss.str(); - } + class pruned_transaction { + transaction& tx; + public: + pruned_transaction(transaction& tx) : tx(tx) {} + BEGIN_SERIALIZE_OBJECT() + bool r = tx.serialize_base(ar); + if (!r) return false; + END_SERIALIZE() + }; //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) { @@ -226,47 +230,47 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCKS_FAST>(invoke_http_mode::BIN, "/getblocks.bin", req, res, r)) return r; - std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > > bs; + std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = "Failed"; return false; } size_t pruned_size = 0, unpruned_size = 0, ntxes = 0; + res.blocks.reserve(bs.size()); + res.output_indices.reserve(bs.size()); for(auto& bd: bs) { res.blocks.resize(res.blocks.size()+1); - res.blocks.back().block = bd.first; - pruned_size += bd.first.size(); - unpruned_size += bd.first.size(); + res.blocks.back().block = bd.first.first; + pruned_size += bd.first.first.size(); + unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - block b; - if (!parse_and_validate_block_from_blob(bd.first, b)) - { - res.status = "Invalid block"; - return false; - } - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(b.miner_tx), res.output_indices.back().indices.back().indices); - if (!r) + if (!req.no_miner_tx) { - res.status = "Failed"; - return false; + bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices); + if (!r) + { + res.status = "Failed"; + return false; + } } - size_t txidx = 0; ntxes += bd.second.size(); - for (std::list<cryptonote::blobdata>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + res.blocks.back().txs.reserve(bd.second.size()); + res.output_indices.back().indices.reserve(bd.second.size()); + for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { - unpruned_size += i->size(); - res.blocks.back().txs.push_back(std::move(*i)); - i->clear(); - i->shrink_to_fit(); + unpruned_size += i->second.size(); + res.blocks.back().txs.push_back(std::move(i->second)); + i->second.clear(); + i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices); + bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); if (!r) { res.status = "Failed"; @@ -286,7 +290,7 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_ALT_BLOCKS_HASHES>(invoke_http_mode::JON, "/get_alt_blocks_hashes", req, res, r)) return r; - std::list<block> blks; + std::vector<block> blks; if(!m_core.get_alternative_blocks(blks)) { @@ -328,8 +332,8 @@ namespace cryptonote res.status = "Error retrieving block at height " + std::to_string(height); return true; } - std::list<transaction> txs; - std::list<crypto::hash> missed_txs; + std::vector<transaction> txs; + std::vector<crypto::hash> missed_txs; m_core.get_transactions(blk.tx_hashes, txs, missed_txs); res.blocks.resize(res.blocks.size() + 1); res.blocks.back().block = block_to_blob(blk); @@ -363,49 +367,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) - { - PERF_TIMER(on_get_random_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS>(invoke_http_mode::BIN, "/getrandom_outs.bin", req, res, r)) - return r; - - res.status = "Failed"; - - if (m_restricted) - { - if (req.amounts.size() > 100 || req.outs_count > MAX_RESTRICTED_FAKE_OUTS_COUNT) - { - res.status = "Too many outs requested"; - return true; - } - } - - if(!m_core.get_random_outs_for_amounts(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount outs_for_amount; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - std::for_each(res.outs.begin(), res.outs.end(), [&](outs_for_amount& ofa) - { - ss << "[" << ofa.amount << "]:"; - CHECK_AND_ASSERT_MES(ofa.outs.size(), ;, "internal error: ofa.outs.size() is empty for amount " << ofa.amount); - std::for_each(ofa.outs.begin(), ofa.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - }); - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) { PERF_TIMER(on_get_outs_bin); @@ -475,34 +436,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) - { - PERF_TIMER(on_get_random_rct_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS>(invoke_http_mode::BIN, "/getrandom_rctouts.bin", req, res, r)) - return r; - - res.status = "Failed"; - if(!m_core.get_random_rct_outs(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry out_entry; - CHECK_AND_ASSERT_MES(res.outs.size(), true, "internal error: res.outs.size() is empty"); - std::for_each(res.outs.begin(), res.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) { PERF_TIMER(on_get_indexes); @@ -544,8 +477,8 @@ namespace cryptonote } vh.push_back(*reinterpret_cast<const crypto::hash*>(b.data())); } - std::list<crypto::hash> missed_txs; - std::list<transaction> txs; + std::vector<crypto::hash> missed_txs; + std::vector<transaction> txs; bool r = m_core.get_transactions(vh, txs, missed_txs); if(!r) { @@ -566,25 +499,26 @@ namespace cryptonote if(r) { // sort to match original request - std::list<transaction> sorted_txs; + std::vector<transaction> sorted_txs; std::vector<tx_info>::const_iterator i; + unsigned txs_processed = 0; for (const crypto::hash &h: vh) { if (std::find(missed_txs.begin(), missed_txs.end(), h) == missed_txs.end()) { - if (txs.empty()) + if (txs.size() == txs_processed) { res.status = "Failed: internal error - txs is empty"; return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs.front()) != h) + if (get_transaction_hash(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; } - sorted_txs.push_back(std::move(txs.front())); - txs.pop_front(); + sorted_txs.push_back(std::move(txs[txs_processed])); + ++txs_processed; } else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) { @@ -595,7 +529,7 @@ namespace cryptonote return true; } sorted_txs.push_back(tx); - missed_txs.remove(h); + missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); for (const auto &ti: pool_tx_info) @@ -614,7 +548,7 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } - std::list<std::string>::const_iterator txhi = req.txs_hashes.begin(); + std::vector<std::string>::const_iterator txhi = req.txs_hashes.begin(); std::vector<crypto::hash>::const_iterator vhi = vh.begin(); for(auto& tx: txs) { @@ -623,10 +557,11 @@ namespace cryptonote crypto::hash tx_hash = *vhi++; e.tx_hash = *txhi++; - blobdata blob = req.prune ? get_pruned_tx_blob(tx) : t_serializable_object_to_blob(tx); + pruned_transaction pruned_tx{tx}; + blobdata blob = req.prune ? t_serializable_object_to_blob(pruned_tx) : t_serializable_object_to_blob(tx); e.as_hex = string_tools::buff_to_hex_nodelimer(blob); if (req.decode_as_json) - e.as_json = obj_to_json_str(tx); + e.as_json = req.prune ? obj_to_json_str(pruned_tx) : obj_to_json_str(tx); e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end(); if (e.in_pool) { @@ -848,7 +783,13 @@ namespace cryptonote boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - if(!m_core.get_miner().start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + cryptonote::miner &miner= m_core.get_miner(); + if (miner.is_mining()) + { + res.status = "Already mining"; + return true; + } + if(!miner.start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) { res.status = "Failed, mining not started"; LOG_PRINT_L0(res.status); @@ -977,15 +918,17 @@ namespace cryptonote return r; m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, !request_has_rpc_origin || !m_restricted); + for (tx_info& txi : res.transactions) + txi.tx_blob = epee::string_tools::buff_to_hex_nodelimer(txi.tx_blob); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, bool request_has_rpc_origin) { PERF_TIMER(on_get_transaction_pool_hashes); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES>(invoke_http_mode::JON, "/get_transaction_pool_hashes.bin", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN>(invoke_http_mode::JON, "/get_transaction_pool_hashes.bin", req, res, r)) return r; m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !m_restricted); @@ -993,6 +936,22 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin) + { + PERF_TIMER(on_get_transaction_pool_hashes); + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES>(invoke_http_mode::JON, "/get_transaction_pool_hashes", req, res, r)) + return r; + + std::vector<crypto::hash> tx_hashes; + m_core.get_pool_transaction_hashes(tx_hashes, !request_has_rpc_origin || !m_restricted); + res.tx_hashes.reserve(tx_hashes.size()); + for (const crypto::hash &tx_hash: tx_hashes) + res.tx_hashes.push_back(epee::string_tools::pod_to_hex(tx_hash)); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin) { PERF_TIMER(on_get_transaction_pool_stats); @@ -1127,7 +1086,7 @@ namespace cryptonote { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; - LOG_ERROR("Failed to tx pub key in coinbase extra"); + LOG_ERROR("Failed to get tx pub key in coinbase extra"); return false; } res.reserved_offset = slow_memmem((void*)block_blob.data(), block_blob.size(), &tx_pub_key, sizeof(tx_pub_key)); @@ -1209,6 +1168,68 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp) + { + PERF_TIMER(on_generateblocks); + + CHECK_CORE_READY(); + + res.status = CORE_RPC_STATUS_OK; + + if(m_core.get_nettype() != FAKECHAIN) + { + error_resp.code = CORE_RPC_ERROR_CODE_REGTEST_REQUIRED; + error_resp.message = "Regtest required when generating blocks"; + return false; + } + + COMMAND_RPC_GETBLOCKTEMPLATE::request template_req; + COMMAND_RPC_GETBLOCKTEMPLATE::response template_res; + COMMAND_RPC_SUBMITBLOCK::request submit_req; + COMMAND_RPC_SUBMITBLOCK::response submit_res; + + template_req.reserve_size = 1; + template_req.wallet_address = req.wallet_address; + submit_req.push_back(boost::value_initialized<std::string>()); + res.height = m_core.get_blockchain_storage().get_current_blockchain_height(); + + bool r; + + for(size_t i = 0; i < req.amount_of_blocks; i++) + { + r = on_getblocktemplate(template_req, template_res, error_resp); + res.status = template_res.status; + + if (!r) return false; + + blobdata blockblob; + if(!string_tools::parse_hexstr_to_binbuff(template_res.blocktemplate_blob, blockblob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + block b = AUTO_VAL_INIT(b); + if(!parse_and_validate_block_from_blob(blockblob, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + miner::find_nonce_for_given_block(b, template_res.difficulty, template_res.height); + + submit_req.front() = string_tools::buff_to_hex_nodelimer(block_to_blob(b)); + r = on_submitblock(submit_req, submit_res, error_resp); + res.status = submit_res.status; + + if (!r) return false; + + res.height = template_res.height; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ uint64_t core_rpc_server::get_block_reward(const block& blk) { uint64_t reward = 0; @@ -1232,8 +1253,9 @@ namespace cryptonote response.depth = m_core.get_current_blockchain_height() - height - 1; response.hash = string_tools::pod_to_hex(hash); response.difficulty = m_core.get_blockchain_storage().block_difficulty(height); + response.cumulative_difficulty = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height); response.reward = get_block_reward(blk); - response.block_size = m_core.get_blockchain_storage().get_db().get_block_size(height); + response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; return true; @@ -1564,9 +1586,10 @@ namespace cryptonote res.mainnet = m_nettype == MAINNET; res.testnet = m_nettype == TESTNET; res.stagenet = m_nettype == STAGENET; + res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; res.start_time = (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); @@ -1577,6 +1600,8 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); + res.update_available = m_core.is_update_available(); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1655,10 +1680,10 @@ namespace cryptonote PERF_TIMER(on_flush_txpool); bool failed = false; - std::list<crypto::hash> txids; + std::vector<crypto::hash> txids; if (req.txids.empty()) { - std::list<transaction> pool_txs; + std::vector<transaction> pool_txs; bool r = m_core.get_pool_transactions(pool_txs); if (!r) { @@ -1757,14 +1782,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) { - PERF_TIMER(on_get_per_kb_fee_estimate); + PERF_TIMER(on_get_base_fee_estimate); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BASE_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) return r; - res.fee = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.grace_blocks); + res.fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.grace_blocks); + res.quantization_mask = Blockchain::get_fee_quantization_mask(); res.status = CORE_RPC_STATUS_OK; return true; } @@ -1774,10 +1800,22 @@ namespace cryptonote PERF_TIMER(on_get_alternate_chains); try { - std::list<std::pair<Blockchain::block_extended_info, uint64_t>> chains = m_core.get_blockchain_storage().get_alternative_chains(); + std::list<std::pair<Blockchain::block_extended_info, std::vector<crypto::hash>>> chains = m_core.get_blockchain_storage().get_alternative_chains(); for (const auto &i: chains) { - res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second, i.first.cumulative_difficulty}); + res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), i.first.cumulative_difficulty, {}, std::string()}); + res.chains.back().block_hashes.reserve(i.second.size()); + for (const crypto::hash &block_id: i.second) + res.chains.back().block_hashes.push_back(epee::string_tools::pod_to_hex(block_id)); + if (i.first.height < i.second.size()) + { + res.status = "Error finding alternate chain attachment point"; + return true; + } + cryptonote::block main_chain_parent_block; + try { main_chain_parent_block = m_core.get_blockchain_storage().get_db().get_block_from_height(i.first.height - i.second.size()); } + catch (const std::exception &e) { res.status = "Error finding alternate chain attachment point"; return true; } + res.chains.back().main_chain_parent_block = epee::string_tools::pod_to_hex(get_block_hash(main_chain_parent_block)); } res.status = CORE_RPC_STATUS_OK; } @@ -2070,87 +2108,25 @@ namespace cryptonote bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp) { PERF_TIMER(on_get_output_distribution); + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r)) + return r; + try { + // 0 is placeholder for the whole chain + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - static struct D - { - boost::mutex mutex; - std::vector<uint64_t> cached_distribution; - uint64_t cached_from, cached_to, cached_start_height, cached_base; - bool cached; - D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} - } d; - boost::unique_lock<boost::mutex> lock(d.mutex); - - if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req.to_height) - { - res.distributions.push_back({amount, d.cached_start_height, d.cached_distribution, d.cached_base}); - if (req.cumulative) - { - auto &distribution = res.distributions.back().distribution; - distribution[0] += d.cached_base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; - } - continue; - } - - // this is a slow operation, so we have precomputed caches of common cases - bool found = false; - for (const auto &slot: get_output_distribution_cache) - { - if (slot.amount == amount && slot.from_height == req.from_height && slot.to_height == req.to_height) - { - res.distributions.push_back({amount, slot.start_height, slot.distribution, slot.base}); - found = true; - if (req.cumulative) - { - auto &distribution = res.distributions.back().distribution; - distribution[0] += slot.base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; - } - break; - } - } - if (found) - continue; - - std::vector<uint64_t> distribution; - uint64_t start_height, base; - if (!m_core.get_output_distribution(amount, req.from_height, req.to_height, start_height, distribution, base)) + auto data = rpc::RpcHandler::get_output_distribution(m_core, amount, req.from_height, req_to_height, req.cumulative); + if (!data) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get rct distribution"; + error_resp.message = "Failed to get output distribution"; return false; } - if (req.to_height > 0 && req.to_height >= req.from_height) - { - uint64_t offset = std::max(req.from_height, start_height); - if (offset <= req.to_height && req.to_height - offset + 1 < distribution.size()) - distribution.resize(req.to_height - offset + 1); - } - - if (amount == 0) - { - d.cached_from = req.from_height; - d.cached_to = req.to_height; - d.cached_distribution = distribution; - d.cached_start_height = start_height; - d.cached_base = base; - d.cached = true; - } - - if (req.cumulative) - { - distribution[0] += base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; - } - res.distributions.push_back({amount, start_height, std::move(distribution), base}); + res.distributions.push_back({std::move(*data), amount, req.binary}); } } catch (const std::exception &e) diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 324f219f8..3ba882b23 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -87,11 +87,7 @@ namespace cryptonote MAP_URI_AUTO_BIN2("/get_hashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES) - MAP_URI_AUTO_BIN2("/get_random_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) - MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) MAP_URI_AUTO_BIN2("/get_outs.bin", on_get_outs_bin, COMMAND_RPC_GET_OUTPUTS_BIN) - MAP_URI_AUTO_BIN2("/get_random_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) - MAP_URI_AUTO_BIN2("/getrandom_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) MAP_URI_AUTO_JON2("/get_transactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/get_alt_blocks_hashes", on_get_alt_blocks_hashes, COMMAND_RPC_GET_ALT_BLOCKS_HASHES) @@ -107,7 +103,8 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted) MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted) MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL) - MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES) + MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes_bin, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN) + MAP_URI_AUTO_JON2("/get_transaction_pool_hashes", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES) MAP_URI_AUTO_JON2("/get_transaction_pool_stats", on_get_transaction_pool_stats, COMMAND_RPC_GET_TRANSACTION_POOL_STATS) MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted) MAP_URI_AUTO_JON2("/get_info", on_get_info, COMMAND_RPC_GET_INFO) @@ -129,6 +126,7 @@ namespace cryptonote MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) + MAP_JON_RPC_WE_IF("generateblocks", on_generateblocks, COMMAND_RPC_GENERATEBLOCKS, !m_restricted) MAP_JON_RPC_WE("get_last_block_header", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("getlastblockheader", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("get_block_header_by_hash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) @@ -148,7 +146,7 @@ namespace cryptonote MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) MAP_JON_RPC_WE("get_version", on_get_version, COMMAND_RPC_GET_VERSION) MAP_JON_RPC_WE_IF("get_coinbase_tx_sum", on_get_coinbase_tx_sum, COMMAND_RPC_GET_COINBASE_TX_SUM, !m_restricted) - MAP_JON_RPC_WE("get_fee_estimate", on_get_per_kb_fee_estimate, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE) + MAP_JON_RPC_WE("get_fee_estimate", on_get_base_fee_estimate, COMMAND_RPC_GET_BASE_FEE_ESTIMATE) MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains, COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted) MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) @@ -169,10 +167,8 @@ namespace cryptonote bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); - bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res); bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res); - bool on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res); bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); @@ -180,6 +176,7 @@ namespace cryptonote bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res); bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, bool request_has_rpc_origin = true); + bool on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, bool request_has_rpc_origin = true); bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin = true); bool on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin = true); bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); @@ -196,6 +193,7 @@ namespace cryptonote bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp); + bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp); bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp); bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp); @@ -210,7 +208,7 @@ namespace cryptonote bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp); bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp); bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp); - bool on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); + bool on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 70e186848..8e8df7a52 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -33,6 +33,7 @@ #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/difficulty.h" #include "crypto/hash.h" +#include "rpc/rpc_handler.h" namespace cryptonote { @@ -48,8 +49,8 @@ namespace cryptonote // whether they can talk to a given daemon without having to know in // advance which version they will stop working with // Don't go over 32767 for any of these -#define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 20 +#define CORE_RPC_VERSION_MAJOR 2 +#define CORE_RPC_VERSION_MINOR 1 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -83,10 +84,12 @@ namespace cryptonote std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; bool prune; + bool no_miner_tx; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) KV_SERIALIZE(start_height) KV_SERIALIZE(prune) + KV_SERIALIZE_OPT(no_miner_tx, false) END_KV_SERIALIZE_MAP() }; @@ -110,7 +113,7 @@ namespace cryptonote struct response { - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; uint64_t start_height; uint64_t current_height; std::string status; @@ -188,7 +191,7 @@ namespace cryptonote struct response { - std::list<crypto::hash> m_block_ids; + std::vector<crypto::hash> m_block_ids; uint64_t start_height; uint64_t current_height; std::string status; @@ -273,7 +276,7 @@ namespace cryptonote uint64_t total_received; uint64_t total_received_unlocked = 0; // OpenMonero only uint64_t scanned_height; - std::list<transaction> transactions; + std::vector<transaction> transactions; uint64_t blockchain_height; uint64_t scanned_block_height; std::string status; @@ -561,7 +564,7 @@ namespace cryptonote { struct request { - std::list<std::string> txs_hashes; + std::vector<std::string> txs_hashes; bool decode_as_json; bool prune; @@ -598,11 +601,11 @@ namespace cryptonote struct response { // older compatibility stuff - std::list<std::string> txs_as_hex; //transactions blobs as hex (old compat) - std::list<std::string> txs_as_json; //transactions decoded as json (old compat) + std::vector<std::string> txs_as_hex; //transactions blobs as hex (old compat) + std::vector<std::string> txs_as_json; //transactions decoded as json (old compat) // in both old and new - std::list<std::string> missed_tx; //not found transactions + std::vector<std::string> missed_tx; //not found transactions // new style std::vector<entry> txs; @@ -678,50 +681,6 @@ namespace cryptonote }; }; //----------------------------------------------- - struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS - { - struct request - { - std::vector<uint64_t> amounts; - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amounts) - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t global_amount_index; - crypto::public_key out_key; - }; -#pragma pack(pop) - - struct outs_for_amount - { - uint64_t amount; - std::list<out_entry> outs; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - END_KV_SERIALIZE_MAP() - }; - - struct response - { - std::vector<outs_for_amount> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; - //----------------------------------------------- struct get_outputs_out { uint64_t amount; @@ -816,39 +775,6 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; - - struct COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS - { - struct request - { - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t amount; - uint64_t global_amount_index; - crypto::public_key out_key; - rct::key commitment; - }; -#pragma pack(pop) - - struct response - { - std::list<out_entry> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; //----------------------------------------------- struct COMMAND_RPC_SEND_RAW_TX { @@ -857,9 +783,6 @@ namespace cryptonote std::string tx_as_hex; bool do_not_relay; - request() {} - explicit request(const transaction &); - BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_as_hex) KV_SERIALIZE_OPT(do_not_relay, false) @@ -953,10 +876,13 @@ namespace cryptonote bool mainnet; bool testnet; bool stagenet; + std::string nettype; std::string top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; uint64_t free_space; bool offline; @@ -964,6 +890,8 @@ namespace cryptonote std::string bootstrap_daemon_address; uint64_t height_without_bootstrap; bool was_bootstrap_ever_used; + uint64_t database_size; + bool update_available; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) @@ -982,10 +910,13 @@ namespace cryptonote KV_SERIALIZE(mainnet) KV_SERIALIZE(testnet) KV_SERIALIZE(stagenet) + KV_SERIALIZE(nettype) KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE(block_size_limit) + KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0) KV_SERIALIZE(block_size_median) + KV_SERIALIZE_OPT(block_weight_median, (uint64_t)0) KV_SERIALIZE(start_time) KV_SERIALIZE(free_space) KV_SERIALIZE(offline) @@ -993,6 +924,8 @@ namespace cryptonote KV_SERIALIZE(bootstrap_daemon_address) KV_SERIALIZE(height_without_bootstrap) KV_SERIALIZE(was_bootstrap_ever_used) + KV_SERIALIZE(database_size) + KV_SERIALIZE(update_available) END_KV_SERIALIZE_MAP() }; }; @@ -1149,6 +1082,31 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; + + struct COMMAND_RPC_GENERATEBLOCKS + { + struct request + { + uint64_t amount_of_blocks; + std::string wallet_address; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount_of_blocks) + KV_SERIALIZE(wallet_address) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t height; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(height) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; struct block_header_response { @@ -1162,8 +1120,10 @@ namespace cryptonote uint64_t depth; std::string hash; difficulty_type difficulty; + difficulty_type cumulative_difficulty; uint64_t reward; uint64_t block_size; + uint64_t block_weight; uint64_t num_txes; std::string pow_hash; @@ -1178,8 +1138,10 @@ namespace cryptonote KV_SERIALIZE(depth) KV_SERIALIZE(hash) KV_SERIALIZE(difficulty) + KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE(reward) KV_SERIALIZE(block_size) + KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) END_KV_SERIALIZE_MAP() @@ -1420,6 +1382,7 @@ namespace cryptonote std::string id_hash; std::string tx_json; // TODO - expose this data directly uint64_t blob_size; + uint64_t weight; uint64_t fee; std::string max_used_block_id_hash; uint64_t max_used_block_height; @@ -1437,6 +1400,7 @@ namespace cryptonote KV_SERIALIZE(id_hash) KV_SERIALIZE(tx_json) KV_SERIALIZE(blob_size) + KV_SERIALIZE_OPT(weight, (uint64_t)0) KV_SERIALIZE(fee) KV_SERIALIZE(max_used_block_id_hash) KV_SERIALIZE(max_used_block_height) @@ -1487,7 +1451,7 @@ namespace cryptonote }; }; - struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES + struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN { struct request { @@ -1509,9 +1473,31 @@ namespace cryptonote }; }; + struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<std::string> tx_hashes; + bool untrusted; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(tx_hashes) + KV_SERIALIZE(untrusted) + END_KV_SERIALIZE_MAP() + }; + }; + struct tx_backlog_entry { - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t time_in_pool; }; @@ -1565,6 +1551,8 @@ namespace cryptonote std::vector<txpool_histo> histo; uint32_t num_double_spends; + txpool_stats(): bytes_total(0), bytes_min(0), bytes_max(0), bytes_med(0), fee_total(0), oldest(0), txs_total(0), num_failing(0), num_10m(0), num_not_relayed(0), histo_98pc(0), num_double_spends(0) {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(bytes_total) KV_SERIALIZE(bytes_min) @@ -1929,7 +1917,7 @@ namespace cryptonote { struct request { - std::list<std::string> txids; + std::vector<std::string> txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) @@ -2047,7 +2035,7 @@ namespace cryptonote }; }; - struct COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE + struct COMMAND_RPC_GET_BASE_FEE_ESTIMATE { struct request { @@ -2062,11 +2050,13 @@ namespace cryptonote { std::string status; uint64_t fee; + uint64_t quantization_mask; bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(fee) + KV_SERIALIZE_OPT(quantization_mask, (uint64_t)1) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; @@ -2086,12 +2076,16 @@ namespace cryptonote uint64_t height; uint64_t length; uint64_t difficulty; + std::vector<std::string> block_hashes; + std::string main_chain_parent_block; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block_hash) KV_SERIALIZE(height) KV_SERIALIZE(length) KV_SERIALIZE(difficulty) + KV_SERIALIZE(block_hashes) + KV_SERIALIZE(main_chain_parent_block) END_KV_SERIALIZE_MAP() }; @@ -2146,7 +2140,7 @@ namespace cryptonote { struct request { - std::list<std::string> txids; + std::vector<std::string> txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) @@ -2227,27 +2221,32 @@ namespace cryptonote uint64_t from_height; uint64_t to_height; bool cumulative; + bool binary; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amounts) KV_SERIALIZE_OPT(from_height, (uint64_t)0) KV_SERIALIZE_OPT(to_height, (uint64_t)0) KV_SERIALIZE_OPT(cumulative, false) + KV_SERIALIZE_OPT(binary, true) END_KV_SERIALIZE_MAP() }; struct distribution { + rpc::output_distribution_data data; uint64_t amount; - uint64_t start_height; - std::vector<uint64_t> distribution; - uint64_t base; + bool binary; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) - KV_SERIALIZE(start_height) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) - KV_SERIALIZE(base) + KV_SERIALIZE_N(data.start_height, "start_height") + KV_SERIALIZE(binary) + if (this_ref.binary) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + else + KV_SERIALIZE_N(data.distribution, "distribution") + KV_SERIALIZE_N(data.base, "base") END_KV_SERIALIZE_MAP() }; @@ -2255,10 +2254,12 @@ namespace cryptonote { std::string status; std::vector<distribution> distributions; + bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(distributions) + KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; }; diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 69caaa6a6..5a754749f 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -42,5 +42,6 @@ #define CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB_SIZE -10 #define CORE_RPC_ERROR_CODE_UNSUPPORTED_RPC -11 #define CORE_RPC_ERROR_CODE_MINING_TO_SUBADDRESS -12 +#define CORE_RPC_ERROR_CODE_REGTEST_REQUIRED -13 diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 39f169cdf..8822bd378 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -50,9 +50,9 @@ namespace rpc void DaemonHandler::handle(const GetBlocksFast::Request& req, GetBlocksFast::Response& res) { - std::list<std::pair<blobdata, std::list<blobdata> > > blocks; + std::vector<std::pair<std::pair<blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, blobdata> > > > blocks; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = Message::STATUS_FAILED; res.error_details = "core::find_blockchain_supplement() returned false"; @@ -62,9 +62,6 @@ namespace rpc res.blocks.resize(blocks.size()); res.output_indices.resize(blocks.size()); - //TODO: really need to switch uses of std::list to std::vector unless - // it's a huge performance concern - auto it = blocks.begin(); uint64_t block_count = 0; @@ -72,7 +69,7 @@ namespace rpc { cryptonote::rpc::block_with_transactions& bwt = res.blocks[block_count]; - if (!parse_and_validate_block_from_blob(it->first, bwt.block)) + if (!parse_and_validate_block_from_blob(it->first.first, bwt.block)) { res.blocks.clear(); res.output_indices.clear(); @@ -89,53 +86,45 @@ namespace rpc res.error_details = "incorrect number of transactions retrieved for block"; return; } - std::list<transaction> txs; - for (const auto& blob : it->second) - { - txs.resize(txs.size() + 1); - if (!parse_and_validate_tx_from_blob(blob, txs.back())) - { - res.blocks.clear(); - res.output_indices.clear(); - res.status = Message::STATUS_FAILED; - res.error_details = "failed retrieving a requested transaction"; - return; - } - } cryptonote::rpc::block_output_indices& indices = res.output_indices[block_count]; // miner tx output indices { cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); } - // assume each block returned is returned with all its transactions - // in the correct order. - auto tx_it = txs.begin(); - for (const crypto::hash& h : bwt.block.tx_hashes) + auto hash_it = bwt.block.tx_hashes.begin(); + bwt.transactions.reserve(it->second.size()); + for (const auto& blob : it->second) { - bwt.transactions.emplace(h, *tx_it); - tx_it++; + bwt.transactions.emplace_back(); + if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back())) + { + res.blocks.clear(); + res.output_indices.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "failed retrieving a requested transaction"; + return; + } cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(h, tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(*hash_it, tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); + ++hash_it; } it++; @@ -163,10 +152,10 @@ namespace rpc void DaemonHandler::handle(const GetTransactions::Request& req, GetTransactions::Response& res) { - std::list<cryptonote::transaction> found_txs; - std::list<crypto::hash> missed_hashes; + std::vector<cryptonote::transaction> found_txs_vec; + std::vector<crypto::hash> missed_vec; - bool r = m_core.get_transactions(req.tx_hashes, found_txs, missed_hashes); + bool r = m_core.get_transactions(req.tx_hashes, found_txs_vec, missed_vec); // TODO: consider fixing core::get_transactions to not hide exceptions if (!r) @@ -176,20 +165,7 @@ namespace rpc return; } - size_t num_found = found_txs.size(); - - // std::list is annoying - std::vector<cryptonote::transaction> found_txs_vec - { - std::make_move_iterator(std::begin(found_txs)), - std::make_move_iterator(std::end(found_txs)) - }; - - std::vector<crypto::hash> missed_vec - { - std::make_move_iterator(std::begin(missed_hashes)), - std::make_move_iterator(std::end(missed_hashes)) - }; + size_t num_found = found_txs_vec.size(); std::vector<uint64_t> heights(num_found); std::vector<bool> in_pool(num_found, false); @@ -204,7 +180,7 @@ namespace rpc // if any missing from blockchain, check in tx pool if (!missed_vec.empty()) { - std::list<cryptonote::transaction> pool_txs; + std::vector<cryptonote::transaction> pool_txs; m_core.get_pool_transactions(pool_txs); @@ -284,44 +260,6 @@ namespace rpc } - //TODO: handle "restricted" RPC - void DaemonHandler::handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res) - { - auto& chain = m_core.get_blockchain_storage(); - - try - { - for (const uint64_t& amount : req.amounts) - { - std::vector<uint64_t> indices = chain.get_random_outputs(amount, req.count); - - outputs_for_amount ofa; - - ofa.resize(indices.size()); - - for (size_t i = 0; i < indices.size(); i++) - { - crypto::public_key key = chain.get_output_key(amount, indices[i]); - ofa[i].amount_index = indices[i]; - ofa[i].key = key; - } - - amount_with_random_outputs amt; - amt.amount = amount; - amt.outputs = ofa; - - res.amounts_with_outputs.push_back(amt); - } - - res.status = Message::STATUS_OK; - } - catch (const std::exception& e) - { - res.status = Message::STATUS_FAILED; - res.error_details = e.what(); - } - } - void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) { auto tx_blob = cryptonote::tx_to_blob(req.tx); @@ -496,7 +434,8 @@ namespace rpc res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); - res.info.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); + res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); res.status = Message::STATUS_OK; @@ -785,12 +724,53 @@ namespace rpc res.status = Message::STATUS_OK; } - void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res) + void DaemonHandler::handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res) { - res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.num_grace_blocks); + res.hard_fork_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); + res.estimated_base_fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); + + if (res.hard_fork_version < HF_VERSION_PER_BYTE_FEE) + { + res.size_scale = 1024; // per KiB fee + res.fee_mask = 1; + } + else + { + res.size_scale = 1; // per byte fee + res.fee_mask = Blockchain::get_fee_quantization_mask(); + } res.status = Message::STATUS_OK; } + void DaemonHandler::handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res) + { + try + { + res.distributions.reserve(req.amounts.size()); + + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); + for (std::uint64_t amount : req.amounts) + { + auto data = get_output_distribution(m_core, amount, req.from_height, req_to_height, req.cumulative); + if (!data) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "Failed to get output distribution"; + return; + } + res.distributions.push_back(output_distribution{std::move(*data), amount, req.cumulative}); + } + res.status = Message::STATUS_OK; + } + catch (const std::exception& e) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = e.what(); + } + } + bool DaemonHandler::getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& header) { block b; @@ -847,7 +827,6 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetTransactions, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, KeyImagesSpent, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetTxGlobalOutputIndices, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetRandomOutputsForAmounts, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, SendRawTx, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetInfo, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StartMining, req_json, resp_message, handle); @@ -866,7 +845,8 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetOutputHistogram, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetOutputKeys, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetRPCVersion, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetPerKBFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetOutputDistribution, req_json, resp_message, handle); // if none of the request types matches if (resp_message == NULL) diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index f43711640..2c8ac3867 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -66,8 +66,6 @@ class DaemonHandler : public RpcHandler void handle(const GetTxGlobalOutputIndices::Request& req, GetTxGlobalOutputIndices::Response& res); - void handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res); - void handle(const SendRawTx::Request& req, SendRawTx::Response& res); void handle(const StartMining::Request& req, StartMining::Response& res); @@ -128,7 +126,9 @@ class DaemonHandler : public RpcHandler void handle(const GetRPCVersion::Request& req, GetRPCVersion::Response& res); - void handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res); + void handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res); + + void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res); std::string handle(const std::string& request); diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index e5fb9781c..fa848cff4 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -41,7 +41,6 @@ const char* const GetHashesFast::name = "get_hashes_fast"; const char* const GetTransactions::name = "get_transactions"; const char* const KeyImagesSpent::name = "key_images_spent"; const char* const GetTxGlobalOutputIndices::name = "get_tx_global_output_indices"; -const char* const GetRandomOutputsForAmounts::name = "get_random_outputs_for_amounts"; const char* const SendRawTx::name = "send_raw_tx"; const char* const StartMining::name = "start_mining"; const char* const StopMining::name = "stop_mining"; @@ -60,7 +59,8 @@ const char* const HardForkInfo::name = "hard_fork_info"; const char* const GetOutputHistogram::name = "get_output_histogram"; const char* const GetOutputKeys::name = "get_output_keys"; const char* const GetRPCVersion::name = "get_rpc_version"; -const char* const GetPerKBFeeEstimate::name = "get_dynamic_per_kb_fee_estimate"; +const char* const GetFeeEstimate::name = "get_dynamic_fee_estimate"; +const char* const GetOutputDistribution::name = "get_output_distribution"; @@ -273,42 +273,6 @@ void GetTxGlobalOutputIndices::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, output_indices, output_indices); } - -rapidjson::Value GetRandomOutputsForAmounts::Request::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); - INSERT_INTO_JSON_OBJECT(val, doc, count, count); - - return val; -} - -void GetRandomOutputsForAmounts::Request::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts, amounts); - GET_FROM_JSON_OBJECT(val, count, count); -} - -rapidjson::Value GetRandomOutputsForAmounts::Response::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts_with_outputs, amounts_with_outputs); - - return val; -} - -void GetRandomOutputsForAmounts::Response::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts_with_outputs, amounts_with_outputs); -} - - rapidjson::Value SendRawTx::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); @@ -862,7 +826,7 @@ void GetRPCVersion::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, version, version); } -rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); @@ -873,27 +837,70 @@ rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc) return val; } -void GetPerKBFeeEstimate::Request::fromJson(rapidjson::Value& val) +void GetFeeEstimate::Request::fromJson(rapidjson::Value& val) { GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks); } -rapidjson::Value GetPerKBFeeEstimate::Response::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, estimated_fee_per_kb, estimated_fee_per_kb); + INSERT_INTO_JSON_OBJECT(val, doc, estimated_base_fee, estimated_base_fee); + INSERT_INTO_JSON_OBJECT(val, doc, fee_mask, fee_mask); + INSERT_INTO_JSON_OBJECT(val, doc, size_scale, size_scale); + INSERT_INTO_JSON_OBJECT(val, doc, hard_fork_version, hard_fork_version); return val; } -void GetPerKBFeeEstimate::Response::fromJson(rapidjson::Value& val) +void GetFeeEstimate::Response::fromJson(rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, estimated_fee_per_kb, estimated_fee_per_kb); + GET_FROM_JSON_OBJECT(val, estimated_base_fee, estimated_base_fee); + GET_FROM_JSON_OBJECT(val, fee_mask, fee_mask); + GET_FROM_JSON_OBJECT(val, size_scale, size_scale); + GET_FROM_JSON_OBJECT(val, hard_fork_version, hard_fork_version); } +rapidjson::Value GetOutputDistribution::Request::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + auto& al = doc.GetAllocator(); + + INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); + INSERT_INTO_JSON_OBJECT(val, doc, from_height, from_height); + INSERT_INTO_JSON_OBJECT(val, doc, to_height, to_height); + INSERT_INTO_JSON_OBJECT(val, doc, cumulative, cumulative); + + return val; +} + +void GetOutputDistribution::Request::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, amounts, amounts); + GET_FROM_JSON_OBJECT(val, from_height, from_height); + GET_FROM_JSON_OBJECT(val, to_height, to_height); + GET_FROM_JSON_OBJECT(val, cumulative, cumulative); +} + +rapidjson::Value GetOutputDistribution::Response::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + auto& al = doc.GetAllocator(); + + INSERT_INTO_JSON_OBJECT(val, doc, status, status); + INSERT_INTO_JSON_OBJECT(val, doc, distributions, distributions); + + return val; +} + +void GetOutputDistribution::Response::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, status, status); + GET_FROM_JSON_OBJECT(val, distributions, distributions); +} } // namespace rpc diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 1495c845f..d2014247c 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -28,6 +28,9 @@ #pragma once +#include <unordered_map> +#include <vector> + #include "message.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "rpc/message_data_structs.h" @@ -62,12 +65,10 @@ class classname \ #define END_RPC_MESSAGE_RESPONSE }; #define END_RPC_MESSAGE_CLASS }; -#define COMMA() , - // NOTE: when using a type with multiple template parameters, // replace any comma in the template specifier with the macro // above, or the preprocessor will eat the comma in a bad way. -#define RPC_MESSAGE_MEMBER(type, name) type name; +#define RPC_MESSAGE_MEMBER(type, name) type name = {} namespace cryptonote @@ -106,7 +107,7 @@ BEGIN_RPC_MESSAGE_CLASS(GetHashesFast); RPC_MESSAGE_MEMBER(uint64_t, start_height); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::list<crypto::hash>, hashes); + RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, hashes); RPC_MESSAGE_MEMBER(uint64_t, start_height); RPC_MESSAGE_MEMBER(uint64_t, current_height); END_RPC_MESSAGE_RESPONSE; @@ -118,7 +119,8 @@ BEGIN_RPC_MESSAGE_CLASS(GetTransactions); RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, tx_hashes); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::unordered_map<crypto::hash COMMA() cryptonote::rpc::transaction_info>, txs); + using txes_map = std::unordered_map<crypto::hash, transaction_info>; + RPC_MESSAGE_MEMBER(txes_map, txs); RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, missed_hashes); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; @@ -407,12 +409,27 @@ BEGIN_RPC_MESSAGE_CLASS(GetRPCVersion); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; -BEGIN_RPC_MESSAGE_CLASS(GetPerKBFeeEstimate); +BEGIN_RPC_MESSAGE_CLASS(GetFeeEstimate); BEGIN_RPC_MESSAGE_REQUEST; RPC_MESSAGE_MEMBER(uint64_t, num_grace_blocks); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(uint64_t, estimated_fee_per_kb); + RPC_MESSAGE_MEMBER(uint64_t, estimated_base_fee); + RPC_MESSAGE_MEMBER(uint64_t, fee_mask); + RPC_MESSAGE_MEMBER(uint32_t, size_scale); + RPC_MESSAGE_MEMBER(uint8_t, hard_fork_version); + END_RPC_MESSAGE_RESPONSE; +END_RPC_MESSAGE_CLASS; + +BEGIN_RPC_MESSAGE_CLASS(GetOutputDistribution); + BEGIN_RPC_MESSAGE_REQUEST; + RPC_MESSAGE_MEMBER(std::vector<uint64_t>, amounts); + RPC_MESSAGE_MEMBER(uint64_t, from_height); + RPC_MESSAGE_MEMBER(uint64_t, to_height); + RPC_MESSAGE_MEMBER(bool, cumulative); + END_RPC_MESSAGE_REQUEST; + BEGIN_RPC_MESSAGE_RESPONSE; + RPC_MESSAGE_MEMBER(std::vector<output_distribution>, distributions); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; diff --git a/src/rpc/get_output_distribution_cache.h b/src/rpc/get_output_distribution_cache.h deleted file mode 100644 index 6495e7d4c..000000000 --- a/src/rpc/get_output_distribution_cache.h +++ /dev/null @@ -1,113 +0,0 @@ -static const struct -{ - uint64_t amount; - uint64_t from_height; - uint64_t to_height; - uint64_t start_height; - uint64_t base; - std::vector<uint64_t> distribution; -} -get_output_distribution_cache[] = -{ - { - 0, - 1544704, - 1546001, - 1544704, - 5143500, - { - 5, 38, 37, 33, 39, 7, 1, 1, 5, 9, 7, 5, 17, 5, 3, 9, 3, 17, 5, 17, 1, 1, 15, 13, 3, 10, 5, 3, 34, 1, 45, 7, - 5, 17, 5, 22, 3, 1, 17, 16, 5, 1, 3, 43, 5, 13, 3, 23, 9, 7, 9, 13, 1, 11, 1, 17, 1, 3, 16, 11, 5, 11, 7, 7, - 33, 11, 7, 1, 5, 1, 21, 19, 1, 17, 1, 49, 17, 3, 3, 9, 35, 46, 46, 39, 26, 33, 21, 3, 23, 3, 9, 37, 1, 33, 11, 32, - 1, 13, 16, 12, 3, 21, 1, 18, 3, 19, 1, 25, 5, 3, 18, 7, 17, 5, 9, 15, 7, 7, 11, 9, 9, 17, 5, 16, 1, 3, 13, 3, - 5, 5, 5, 13, 5, 9, 5, 13, 3, 17, 15, 36, 13, 3, 20, 12, 6, 23, 17, 10, 22, 23, 1, 7, 21, 6, 23, 1, 3, 19, 13, 1, - 3, 43, 35, 13, 1, 31, 7, 3, 17, 1, 15, 5, 11, 15, 24, 1, 18, 13, 5, 15, 1, 29, 3, 3, 13, 3, 15, 7, 17, 3, 1, 1, - 17, 1, 1, 45, 39, 27, 45, 46, 34, 7, 3, 3, 9, 3, 3, 11, 7, 5, 9, 25, 19, 3, 33, 1, 5, 17, 1, 45, 4, 1, 45, 11, - 44, 32, 3, 1, 3, 7, 17, 15, 5, 45, 35, 41, 1, 35, 3, 3, 19, 1, 9, 17, 29, 29, 3, 1, 13, 1, 3, 47, 21, 13, 7, 1, - 7, 5, 1, 11, 1, 40, 9, 7, 3, 3, 13, 25, 1, 47, 5, 7, 3, 7, 31, 40, 34, 6, 3, 15, 3, 31, 5, 13, 27, 9, 12, 21, - 3, 1, 19, 1, 19, 5, 47, 49, 47, 42, 50, 34, 29, 23, 1, 5, 9, 16, 11, 7, 1, 19, 7, 5, 1, 15, 1, 1, 9, 13, 9, 5, - 27, 3, 3, 29, 1, 33, 3, 9, 5, 35, 5, 1, 17, 7, 3, 39, 3, 28, 19, 1, 1, 9, 1, 3, 27, 1, 37, 3, 1, 1, 16, 3, - 25, 11, 5, 3, 33, 45, 17, 11, 7, 22, 9, 1, 5, 5, 5, 15, 1, 15, 9, 7, 11, 13, 37, 49, 46, 38, 11, 1, 25, 1, 13, 18, - 3, 7, 39, 3, 37, 19, 35, 3, 1, 3, 19, 1, 3, 15, 21, 3, 27, 1, 45, 48, 1, 13, 29, 9, 1, 1, 46, 43, 5, 15, 3, 7, - 29, 26, 5, 5, 21, 37, 17, 21, 3, 13, 1, 5, 1, 17, 5, 31, 13, 1, 11, 3, 46, 9, 3, 7, 1, 1, 41, 1, 21, 1, 5, 12, - 7, 13, 9, 25, 1, 47, 47, 48, 48, 48, 48, 48, 47, 48, 45, 45, 33, 52, 50, 46, 45, 47, 35, 41, 38, 35, 42, 38, 34, 41, 39, 35, - 51, 51, 45, 43, 49, 52, 53, 45, 42, 46, 37, 53, 49, 41, 46, 49, 46, 47, 48, 37, 41, 33, 43, 38, 15, 3, 3, 27, 11, 5, 23, 13, - 1, 1, 37, 3, 15, 3, 30, 13, 3, 45, 12, 3, 5, 11, 1, 1, 21, 9, 11, 19, 1, 1, 1, 25, 5, 21, 3, 1, 32, 44, 3, 33, - 11, 7, 5, 23, 1, 37, 47, 48, 48, 48, 48, 48, 48, 46, 47, 47, 50, 45, 49, 50, 46, 47, 49, 45, 51, 49, 50, 49, 49, 46, 47, 48, - 46, 48, 46, 50, 46, 43, 46, 46, 48, 47, 46, 47, 45, 49, 46, 43, 50, 45, 45, 49, 45, 48, 45, 49, 48, 45, 45, 51, 45, 51, 45, 46, - 52, 45, 45, 51, 51, 52, 44, 45, 52, 50, 50, 46, 47, 51, 51, 46, 47, 47, 47, 50, 47, 51, 48, 49, 51, 50, 48, 48, 48, 50, 49, 49, - 52, 52, 49, 50, 49, 49, 49, 51, 52, 49, 52, 50, 49, 47, 29, 15, 39, 17, 31, 5, 40, 5, 18, 23, 25, 7, 35, 26, 5, 31, 49, 22, - 3, 17, 7, 49, 7, 49, 47, 12, 44, 46, 36, 15, 3, 1, 47, 13, 35, 40, 5, 21, 19, 39, 21, 33, 31, 29, 1, 1, 37, 1, 15, 47, - 7, 7, 47, 41, 13, 3, 47, 31, 9, 33, 13, 43, 29, 5, 1, 9, 33, 7, 27, 15, 15, 25, 5, 43, 22, 31, 7, 1, 47, 1, 15, 27, - 3, 27, 45, 15, 1, 36, 17, 1, 23, 39, 38, 45, 7, 7, 19, 7, 11, 47, 33, 16, 3, 45, 45, 45, 9, 27, 3, 3, 21, 3, 7, 21, - 7, 3, 43, 1, 17, 1, 45, 37, 46, 5, 5, 13, 46, 40, 48, 48, 45, 34, 1, 46, 19, 25, 9, 7, 47, 23, 37, 31, 3, 25, 13, 46, - 31, 25, 5, 46, 35, 52, 11, 23, 27, 4, 15, 11, 11, 11, 9, 34, 7, 9, 15, 34, 9, 27, 37, 28, 25, 45, 13, 30, 5, 25, 15, 7, - 3, 19, 27, 1, 7, 11, 1, 32, 3, 45, 11, 9, 21, 25, 9, 13, 13, 1, 7, 1, 33, 11, 5, 3, 3, 27, 27, 5, 3, 37, 17, 17, - 3, 7, 5, 13, 1, 3, 44, 45, 26, 25, 1, 13, 3, 13, 3, 11, 1, 11, 7, 45, 3, 3, 1, 43, 1, 19, 3, 1, 15, 5, 39, 7, - 7, 1, 9, 1, 11, 19, 3, 35, 29, 7, 15, 11, 40, 7, 44, 38, 34, 7, 9, 7, 1, 27, 1, 9, 5, 45, 1, 21, 3, 1, 5, 9, - 3, 21, 23, 33, 3, 1, 7, 3, 3, 7, 41, 9, 7, 1, 5, 31, 9, 7, 1, 1, 11, 41, 51, 20, 9, 47, 39, 17, 9, 35, 1, 41, - 1, 19, 1, 19, 15, 1, 13, 5, 23, 15, 9, 15, 17, 1, 15, 27, 33, 31, 29, 7, 13, 1, 5, 45, 5, 1, 1, 11, 1, 13, 3, 7, - 9, 1, 13, 39, 3, 33, 5, 3, 7, 7, 5, 29, 11, 1, 7, 1, 15, 3, 13, 3, 15, 3, 3, 1, 5, 1, 9, 1, 44, 49, 24, 25, - 1, 1, 34, 22, 7, 5, 5, 5, 10, 9, 13, 3, 9, 1, 9, 19, 7, 43, 48, 7, 11, 7, 3, 3, 7, 21, 1, 1, 3, 3, 11, 31, - 1, 1, 13, 22, 23, 7, 27, 9, 3, 3, 21, 1, 35, 21, 9, 11, 13, 39, 1, 3, 7, 23, 3, 28, 3, 45, 47, 38, 32, 37, 34, 1, - 23, 3, 3, 1, 19, 19, 1, 5, 13, 1, 5, 11, 38, 3, 1, 36, 13, 1, 1, 23, 5, 17, 11, 1, 13, 1, 3, 7, 11, 3, 33, 7, - 19, 5, 5, 1, 1, 3, 5, 41, 1, 3, 25, 1, 7, 7, 9, 3, 11, 3, 13, 5, 7, 1, 3, 9, 1, 1, 43, 47, 47, 47, 17, 7, - 17, 3, 19, 1, 9, 9, 33, 22, 1, 25, 1, 3, 3, 32, 5, 1, 23, 9, 5, 1, 31, 5, 9, 1, 3, 7, 19, 1, 12, 11, 5, 1, - 1, 9, 25, 15, 15, 13, 5, 3, 15, 1, 17, 1, 1, 5, 5, 41, 11, 15, 7, 3, 21, 21, 35, 22, 46, 35, 3, 27, 5, 3, 45, 22, - 27, 1, 19, 9, 1, 25, 1, 29, 3, 5, 25, 17, 27, 5, 19, 5, 25, 7, 19, 1, 9, 21, 3, 7, 29, 27, 17, 3, 3, 15, 7, 19, - 5, 25, 1, 23, 45, 4, 31, 1, 37, 14, 29, 3, 29, 1, 23, 29, 19, 11, 1, 13, 13, 9, 1, 25, 1, 33, 1, 37, 37, 23, 7, 21, - 7, 3, 13, 7, 3, 7, 21, 11, 9, 1, 31, 3, 1, 7, 39, 46, 3, 30, - }, - }, - { - 0, - 1562704, - 1564001, - 1562704, - 5521986, - { - 35, 45, 23, 3, 44, 47, 44, 3, 17, 35, 7, 11, 7, 29, 43, 27, 11, 7, 5, 31, 13, 9, 45, 45, 7, 42, 17, 15, 19, 11, 45, 19, - 45, 46, 45, 46, 32, 34, 43, 34, 46, 47, 45, 30, 17, 45, 46, 36, 35, 38, 19, 9, 23, 17, 3, 19, 31, 41, 35, 24, 15, 45, 15, 5, - 11, 5, 19, 11, 11, 7, 15, 19, 45, 34, 7, 7, 29, 1, 9, 36, 7, 44, 45, 33, 25, 8, 17, 7, 44, 43, 48, 45, 42, 46, 40, 44, - 1, 43, 45, 46, 46, 35, 19, 19, 23, 5, 13, 19, 7, 16, 9, 3, 25, 34, 3, 27, 9, 39, 3, 43, 21, 1, 45, 45, 39, 25, 23, 13, - 39, 39, 3, 45, 43, 46, 44, 40, 39, 33, 45, 47, 38, 45, 45, 39, 47, 47, 45, 46, 35, 45, 43, 47, 45, 40, 34, 42, 42, 48, 49, 47, - 47, 48, 47, 45, 43, 48, 37, 48, 41, 45, 48, 34, 42, 44, 9, 19, 27, 1, 47, 47, 43, 25, 29, 5, 5, 21, 39, 35, 43, 37, 13, 45, - 25, 31, 26, 47, 45, 23, 23, 39, 32, 25, 44, 47, 35, 47, 15, 17, 7, 9, 5, 35, 31, 3, 45, 47, 46, 13, 17, 48, 45, 9, 13, 45, - 45, 31, 1, 53, 44, 46, 39, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 47, 47, 47, 47, 47, 47, 47, 53, 49, 45, 45, 50, - 50, 27, 43, 46, 47, 46, 45, 48, 36, 42, 42, 46, 45, 47, 41, 45, 43, 47, 38, 48, 47, 33, 11, 45, 46, 34, 42, 32, 3, 45, 37, 45, - 15, 3, 45, 29, 31, 9, 5, 3, 27, 5, 21, 25, 7, 15, 46, 34, 5, 3, 17, 3, 9, 13, 7, 11, 3, 1, 34, 13, 7, 45, 33, 26, - 9, 5, 9, 41, 43, 34, 3, 35, 3, 17, 37, 11, 17, 3, 15, 27, 15, 45, 46, 13, 26, 16, 11, 7, 5, 45, 38, 45, 45, 22, 44, 44, - 43, 6, 11, 35, 15, 44, 17, 27, 13, 3, 40, 5, 9, 7, 35, 19, 5, 5, 1, 28, 33, 15, 45, 5, 29, 3, 31, 12, 5, 32, 24, 3, - 23, 13, 47, 45, 42, 46, 39, 21, 21, 1, 44, 44, 47, 41, 5, 1, 11, 36, 20, 5, 45, 39, 45, 45, 44, 45, 32, 22, 40, 11, 38, 1, - 45, 46, 37, 9, 23, 9, 15, 44, 7, 16, 38, 46, 11, 14, 24, 19, 19, 7, 26, 25, 45, 37, 17, 1, 35, 1, 3, 28, 3, 11, 22, 13, - 3, 1, 7, 38, 5, 3, 1, 26, 1, 3, 43, 44, 46, 45, 21, 11, 1, 44, 27, 1, 11, 26, 10, 44, 45, 45, 45, 47, 47, 45, 48, 45, - 38, 9, 5, 44, 46, 27, 3, 12, 1, 11, 3, 44, 43, 1, 5, 2, 46, 17, 13, 19, 1, 12, 7, 23, 1, 17, 6, 13, 3, 5, 27, 7, - 46, 36, 19, 25, 1, 1, 3, 8, 15, 3, 45, 45, 45, 37, 6, 15, 3, 5, 38, 14, 41, 1, 13, 45, 45, 39, 44, 29, 43, 48, 51, 50, - 37, 5, 17, 46, 47, 31, 5, 42, 49, 38, 39, 24, 7, 11, 44, 35, 21, 6, 15, 5, 47, 13, 28, 45, 34, 27, 24, 15, 35, 13, 7, 25, - 43, 13, 14, 5, 3, 5, 46, 45, 45, 35, 5, 21, 28, 3, 13, 4, 30, 19, 45, 45, 40, 37, 5, 40, 17, 9, 3, 9, 13, 4, 17, 33, - 44, 39, 17, 45, 28, 5, 11, 45, 19, 45, 21, 44, 31, 43, 49, 48, 15, 3, 1, 44, 45, 43, 11, 1, 1, 27, 43, 45, 39, 3, 1, 3, - 5, 31, 1, 43, 23, 19, 7, 5, 45, 31, 11, 7, 3, 9, 5, 7, 13, 43, 47, 45, 46, 47, 14, 3, 25, 45, 7, 17, 32, 21, 3, 17, - 5, 11, 31, 40, 45, 20, 45, 45, 32, 38, 47, 38, 45, 41, 49, 30, 45, 5, 36, 31, 22, 3, 46, 45, 13, 21, 23, 5, 46, 45, 33, 19, - 25, 1, 7, 13, 5, 44, 23, 29, 23, 24, 7, 5, 37, 13, 29, 13, 7, 17, 7, 43, 3, 21, 7, 44, 1, 19, 15, 9, 34, 43, 26, 3, - 17, 5, 6, 5, 7, 3, 33, 35, 43, 41, 48, 47, 30, 45, 19, 7, 5, 33, 11, 34, 25, 1, 21, 11, 29, 7, 1, 4, 5, 10, 14, 3, - 44, 11, 47, 45, 33, 3, 9, 7, 40, 23, 9, 1, 3, 1, 7, 5, 9, 9, 6, 11, 45, 41, 45, 19, 5, 11, 10, 39, 9, 19, 3, 11, - 43, 42, 1, 13, 35, 5, 32, 7, 5, 5, 43, 37, 3, 32, 17, 3, 23, 1, 13, 45, 17, 1, 21, 45, 43, 46, 49, 47, 45, 30, 9, 31, - 19, 42, 19, 44, 17, 14, 19, 25, 1, 7, 5, 45, 19, 13, 7, 3, 1, 1, 9, 21, 37, 9, 11, 1, 3, 37, 27, 13, 5, 21, 33, 3, - 27, 9, 27, 1, 39, 1, 46, 21, 10, 13, 21, 40, 22, 35, 41, 9, 33, 3, 17, 8, 45, 46, 42, 46, 47, 47, 47, 48, 48, 47, 43, 48, - 37, 39, 50, 35, 3, 40, 45, 40, 46, 36, 34, 28, 9, 9, 37, 9, 5, 7, 13, 31, 1, 7, 3, 3, 44, 45, 25, 15, 1, 21, 43, 25, - 1, 38, 34, 42, 31, 23, 33, 35, 37, 20, 7, 15, 3, 7, 7, 27, 45, 45, 48, 47, 45, 44, 47, 23, 25, 36, 11, 3, 18, 24, 27, 13, - 41, 13, 5, 5, 7, 19, 15, 7, 5, 14, 45, 45, 37, 1, 5, 17, 21, 41, 17, 37, 53, 41, 45, 45, 45, 45, 45, 45, 45, 45, 43, 47, - 47, 48, 53, 47, 47, 47, 49, 27, 45, 47, 47, 47, 47, 45, 45, 45, 47, 43, 48, 34, 34, 43, 46, 15, 37, 21, 5, 27, 11, 1, 9, 7, - 19, 15, 1, 1, 19, 3, 36, 27, 29, 13, 21, 9, 17, 5, 16, 45, 23, 34, 3, 1, 7, 25, 28, 13, 29, 15, 11, 19, 17, 1, 27, 23, - 31, 19, 41, 41, 40, 47, 28, 31, 26, 26, 36, 17, 5, 1, 23, 1, 45, 34, 49, 51, 34, 43, 37, 5, 41, 15, 5, 21, 1, 7, 9, 19, - 5, 11, 39, 19, 45, 45, 38, 17, 9, 1, 15, 11, 5, 13, 47, 46, 48, 45, 19, 32, 7, 19, 5, 7, 23, 29, 5, 45, 41, 37, 1, 5, - 27, 5, 5, 7, 19, 9, 1, 35, 48, 38, 38, 39, 42, 43, 21, 23, 43, 41, 7, 3, 7, 13, 1, 46, 47, 46, 23, 46, 45, 25, 7, 9, - 21, 7, 41, 13, 20, 1, 21, 15, 37, 5, 40, 45, 45, 5, 45, 46, 15, 33, 46, 12, 13, 7, 24, 7, 5, 30, 7, 46, 13, 8, 44, 35, - 45, 33, 40, 36, 47, 47, 29, 43, 36, 43, 45, 42, 36, 19, 7, 7, 43, 3, 44, 25, 48, 29, 11, 45, 30, 1, 17, 13, 25, 1, 48, 45, - 45, 45, 44, 49, 37, 9, 21, 17, 15, 7, 15, 25, 1, 1, 9, 43, 33, 11, 3, 29, 45, 45, 9, 7, 7, 27, 47, 45, 47, 48, 45, 47, - 26, 1, 43, 15, 45, 17, 1, 5, 35, 31, 9, 3, 9, 19, 9, 21, 43, 5, 27, 1, 5, 9, 4, 34, 19, 3, 7, 11, 45, 46, 45, 45, - 46, 47, 47, 44, 45, 43, 27, 9, 17, 15, 19, 44, 45, 46, 47, 47, 45, 45, - } - } -}; - diff --git a/src/rpc/message.h b/src/rpc/message.h index 16b8e92fc..56087b998 100644 --- a/src/rpc/message.h +++ b/src/rpc/message.h @@ -65,7 +65,7 @@ namespace rpc static const char* STATUS_BAD_REQUEST; static const char* STATUS_BAD_JSON; - Message() : status(STATUS_OK) { } + Message() : status(STATUS_OK), rpc_version(0) { } virtual ~Message() { } diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index 17ae9629f..3b56aff15 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -31,6 +31,7 @@ #include "crypto/hash.h" #include "cryptonote_basic/cryptonote_basic.h" #include "ringct/rctSigs.h" +#include "rpc/rpc_handler.h" #include <unordered_map> #include <vector> @@ -44,7 +45,7 @@ namespace rpc struct block_with_transactions { cryptonote::block block; - std::unordered_map<crypto::hash, cryptonote::transaction> transactions; + std::vector<cryptonote::transaction> transactions; }; typedef std::vector<uint64_t> tx_output_indices; @@ -85,6 +86,7 @@ namespace rpc cryptonote::transaction tx; crypto::hash tx_hash; uint64_t blob_size; + uint64_t weight; uint64_t fee; crypto::hash max_used_block_hash; uint64_t max_used_block_height; @@ -181,12 +183,22 @@ namespace rpc bool mainnet; bool testnet; bool stagenet; + std::string nettype; crypto::hash top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; + uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; }; + struct output_distribution + { + output_distribution_data data; + uint64_t amount; + bool cumulative; + }; } // namespace rpc } // namespace cryptonote diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index d4a6138ba..60c78480a 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -82,11 +82,17 @@ namespace cryptonote } } - if (command_line::has_arg(vm, arg.rpc_login)) + const char *env_rpc_login = nullptr; + const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login); + const bool use_rpc_env = !has_rpc_arg && (env_rpc_login = getenv("RPC_LOGIN")) != nullptr && strlen(env_rpc_login) > 0; + boost::optional<tools::login> login{}; + if (has_rpc_arg || use_rpc_env) { - config.login = tools::login::parse(command_line::get_arg(vm, arg.rpc_login), true, [](bool verify) { - return tools::password_container::prompt(verify, "RPC server password"); - }); + config.login = tools::login::parse( + has_rpc_arg ? command_line::get_arg(vm, arg.rpc_login) : std::string(env_rpc_login), true, [](bool verify) { + return tools::password_container::prompt(verify, "RPC server password"); + }); + if (!config.login) return boost::none; @@ -102,7 +108,7 @@ namespace cryptonote { if (!config.login) { - LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RFC server password --") << arg.rpc_login.name << tr(" cannot be empty")); + LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RPC server password --") << arg.rpc_login.name << tr(" cannot be empty")); return boost::none; } diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp new file mode 100644 index 000000000..d4beb1928 --- /dev/null +++ b/src/rpc/rpc_handler.cpp @@ -0,0 +1,69 @@ + +#include <algorithm> +#include <boost/thread/locks.hpp> +#include <boost/thread/mutex.hpp> + +#include "cryptonote_core/cryptonote_core.h" + +namespace cryptonote +{ +namespace rpc +{ + namespace + { + output_distribution_data + process_distribution(bool cumulative, std::uint64_t start_height, std::vector<std::uint64_t> distribution, std::uint64_t base) + { + if (!cumulative && !distribution.empty()) + { + for (std::size_t n = distribution.size() - 1; 0 < n; --n) + distribution[n] -= distribution[n - 1]; + distribution[0] -= base; + } + + return {std::move(distribution), start_height, base}; + } + } + + boost::optional<output_distribution_data> + RpcHandler::get_output_distribution(core& src, std::uint64_t amount, std::uint64_t from_height, std::uint64_t to_height, bool cumulative) + { + static struct D + { + boost::mutex mutex; + std::vector<std::uint64_t> cached_distribution; + std::uint64_t cached_from, cached_to, cached_start_height, cached_base; + bool cached; + D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} + } d; + const boost::unique_lock<boost::mutex> lock(d.mutex); + + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height) + return process_distribution(cumulative, d.cached_start_height, d.cached_distribution, d.cached_base); + + std::vector<std::uint64_t> distribution; + std::uint64_t start_height, base; + if (!src.get_output_distribution(amount, from_height, to_height, start_height, distribution, base)) + return boost::none; + + if (to_height > 0 && to_height >= from_height) + { + const std::uint64_t offset = std::max(from_height, start_height); + if (offset <= to_height && to_height - offset + 1 < distribution.size()) + distribution.resize(to_height - offset + 1); + } + + if (amount == 0) + { + d.cached_from = from_height; + d.cached_to = to_height; + d.cached_distribution = distribution; + d.cached_start_height = start_height; + d.cached_base = base; + d.cached = true; + } + + return process_distribution(cumulative, start_height, std::move(distribution), base); + } +} // rpc +} // cryptonote diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index 64bade5a8..3cccef78a 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -28,24 +28,35 @@ #pragma once +#include <boost/optional/optional.hpp> +#include <cstdint> #include <string> +#include <vector> namespace cryptonote { +class core; namespace rpc { +struct output_distribution_data +{ + std::vector<std::uint64_t> distribution; + std::uint64_t start_height; + std::uint64_t base; +}; class RpcHandler { public: + RpcHandler() { } + virtual ~RpcHandler() { } virtual std::string handle(const std::string& request) = 0; - RpcHandler() { } - - virtual ~RpcHandler() { } + static boost::optional<output_distribution_data> + get_output_distribution(core& src, std::uint64_t amount, std::uint64_t from_height, std::uint64_t to_height, bool cumulative); }; diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index 3aee8c4c7..edd3e6669 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -104,6 +104,10 @@ bool ZmqServer::addTCPSocket(std::string address, std::string port) rep_socket->setsockopt(ZMQ_RCVTIMEO, &DEFAULT_RPC_RECV_TIMEOUT_MS, sizeof(DEFAULT_RPC_RECV_TIMEOUT_MS)); + if (address.empty()) + address = "*"; + if (port.empty()) + port = "*"; std::string bind_address = addr_prefix + address + std::string(":") + port; rep_socket->bind(bind_address.c_str()); } diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index a7fb58ee4..7980e8953 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -28,6 +28,8 @@ #include "json_object.h" +#include <boost/range/adaptor/transformed.hpp> +#include <boost/variant/apply_visitor.hpp> #include <limits> #include <type_traits> #include "string_tools.h" @@ -219,11 +221,11 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::transaction& tx, ra INSERT_INTO_JSON_OBJECT(val, doc, version, tx.version); INSERT_INTO_JSON_OBJECT(val, doc, unlock_time, tx.unlock_time); - INSERT_INTO_JSON_OBJECT(val, doc, vin, tx.vin); - INSERT_INTO_JSON_OBJECT(val, doc, vout, tx.vout); + INSERT_INTO_JSON_OBJECT(val, doc, inputs, tx.vin); + INSERT_INTO_JSON_OBJECT(val, doc, outputs, tx.vout); INSERT_INTO_JSON_OBJECT(val, doc, extra, tx.extra); INSERT_INTO_JSON_OBJECT(val, doc, signatures, tx.signatures); - INSERT_INTO_JSON_OBJECT(val, doc, rct_signatures, tx.rct_signatures); + INSERT_INTO_JSON_OBJECT(val, doc, ringct, tx.rct_signatures); } @@ -236,11 +238,11 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx) GET_FROM_JSON_OBJECT(val, tx.version, version); GET_FROM_JSON_OBJECT(val, tx.unlock_time, unlock_time); - GET_FROM_JSON_OBJECT(val, tx.vin, vin); - GET_FROM_JSON_OBJECT(val, tx.vout, vout); + GET_FROM_JSON_OBJECT(val, tx.vin, inputs); + GET_FROM_JSON_OBJECT(val, tx.vout, outputs); GET_FROM_JSON_OBJECT(val, tx.extra, extra); GET_FROM_JSON_OBJECT(val, tx.signatures, signatures); - GET_FROM_JSON_OBJECT(val, tx.rct_signatures, rct_signatures); + GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct); } void toJsonValue(rapidjson::Document& doc, const cryptonote::block& b, rapidjson::Value& val) @@ -277,26 +279,31 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::txin_v& txin, rapid { val.SetObject(); - if (txin.type() == typeid(cryptonote::txin_gen)) + struct add_input { - val.AddMember("type", "txin_gen", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_gen>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_script)) - { - val.AddMember("type", "txin_to_script", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_script>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_scripthash)) - { - val.AddMember("type", "txin_to_scripthash", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_scripthash>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_key)) - { - val.AddMember("type", "txin_to_key", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_key>(txin)); - } + using result_type = void; + + rapidjson::Document& doc; + rapidjson::Value& val; + + void operator()(cryptonote::txin_to_key const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_key, input); + } + void operator()(cryptonote::txin_gen const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, gen, input); + } + void operator()(cryptonote::txin_to_script const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_script, input); + } + void operator()(cryptonote::txin_to_scripthash const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_scripthash, input); + } + }; + boost::apply_visitor(add_input{doc, val}, txin); } @@ -307,31 +314,37 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin) throw WRONG_TYPE("json object"); } - OBJECT_HAS_MEMBER_OR_THROW(val, "type") - OBJECT_HAS_MEMBER_OR_THROW(val, "value") - if (val["type"]== "txin_gen") + if (val.MemberCount() != 1) { - cryptonote::txin_gen tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; + throw MISSING_KEY("Invalid input object"); } - else if (val["type"]== "txin_to_script") - { - cryptonote::txin_to_script tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; - } - else if (val["type"] == "txin_to_scripthash") - { - cryptonote::txin_to_scripthash tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; - } - else if (val["type"] == "txin_to_key") + + for (auto const& elem : val.GetObject()) { - cryptonote::txin_to_key tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; + if (elem.name == "to_key") + { + cryptonote::txin_to_key tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "gen") + { + cryptonote::txin_gen tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "to_script") + { + cryptonote::txin_to_script tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "to_scripthash") + { + cryptonote::txin_to_scripthash tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } } } @@ -405,7 +418,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::txin_to_key& txin, INSERT_INTO_JSON_OBJECT(val, doc, amount, txin.amount); INSERT_INTO_JSON_OBJECT(val, doc, key_offsets, txin.key_offsets); - INSERT_INTO_JSON_OBJECT(val, doc, k_image, txin.k_image); + INSERT_INTO_JSON_OBJECT(val, doc, key_image, txin.k_image); } @@ -418,58 +431,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin) GET_FROM_JSON_OBJECT(val, txin.amount, amount); GET_FROM_JSON_OBJECT(val, txin.key_offsets, key_offsets); - GET_FROM_JSON_OBJECT(val, txin.k_image, k_image); -} - -void toJsonValue(rapidjson::Document& doc, const cryptonote::txout_target_v& txout, rapidjson::Value& val) -{ - val.SetObject(); - - if (txout.type() == typeid(cryptonote::txout_to_script)) - { - val.AddMember("type", "txout_to_script", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_script>(txout)); - } - else if (txout.type() == typeid(cryptonote::txout_to_scripthash)) - { - val.AddMember("type", "txout_to_scripthash", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_scripthash>(txout)); - } - else if (txout.type() == typeid(cryptonote::txout_to_key)) - { - val.AddMember("type", "txout_to_key", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_key>(txout)); - } -} - - -void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_target_v& txout) -{ - if (!val.IsObject()) - { - throw WRONG_TYPE("json object"); - } - - OBJECT_HAS_MEMBER_OR_THROW(val, "type") - OBJECT_HAS_MEMBER_OR_THROW(val, "value") - if (val["type"]== "txout_to_script") - { - cryptonote::txout_to_script tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } - else if (val["type"] == "txout_to_scripthash") - { - cryptonote::txout_to_scripthash tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } - else if (val["type"] == "txout_to_key") - { - cryptonote::txout_to_key tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } + GET_FROM_JSON_OBJECT(val, txin.k_image, key_image); } void toJsonValue(rapidjson::Document& doc, const cryptonote::txout_to_script& txout, rapidjson::Value& val) @@ -533,7 +495,28 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::tx_out& txout, rapi val.SetObject(); INSERT_INTO_JSON_OBJECT(val, doc, amount, txout.amount); - INSERT_INTO_JSON_OBJECT(val, doc, target, txout.target); + + struct add_output + { + using result_type = void; + + rapidjson::Document& doc; + rapidjson::Value& val; + + void operator()(cryptonote::txout_to_key const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_key, output); + } + void operator()(cryptonote::txout_to_script const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_script, output); + } + void operator()(cryptonote::txout_to_scripthash const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_scripthash, output); + } + }; + boost::apply_visitor(add_output{doc, val}, txout.target); } void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout) @@ -543,8 +526,37 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout) throw WRONG_TYPE("json object"); } - GET_FROM_JSON_OBJECT(val, txout.amount, amount); - GET_FROM_JSON_OBJECT(val, txout.target, target); + if (val.MemberCount() != 2) + { + throw MISSING_KEY("Invalid input object"); + } + + for (auto const& elem : val.GetObject()) + { + if (elem.name == "amount") + { + fromJsonValue(elem.value, txout.amount); + } + + if (elem.name == "to_key") + { + cryptonote::txout_to_key tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + else if (elem.name == "to_script") + { + cryptonote::txout_to_script tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + else if (elem.name == "to_scripthash") + { + cryptonote::txout_to_scripthash tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + } } void toJsonValue(rapidjson::Document& doc, const cryptonote::connection_info& info, rapidjson::Value& val) @@ -617,7 +629,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::block_complete_entr val.SetObject(); INSERT_INTO_JSON_OBJECT(val, doc, block, blk.block); - INSERT_INTO_JSON_OBJECT(val, doc, txs, blk.txs); + INSERT_INTO_JSON_OBJECT(val, doc, transactions, blk.txs); } @@ -629,7 +641,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry } GET_FROM_JSON_OBJECT(val, blk.block, block); - GET_FROM_JSON_OBJECT(val, blk.txs, txs); + GET_FROM_JSON_OBJECT(val, blk.txs, transactions); } void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::block_with_transactions& blk, rapidjson::Value& val) @@ -745,6 +757,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx INSERT_INTO_JSON_OBJECT(val, doc, tx, tx.tx); INSERT_INTO_JSON_OBJECT(val, doc, tx_hash, tx.tx_hash); INSERT_INTO_JSON_OBJECT(val, doc, blob_size, tx.blob_size); + INSERT_INTO_JSON_OBJECT(val, doc, weight, tx.weight); INSERT_INTO_JSON_OBJECT(val, doc, fee, tx.fee); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_hash, tx.max_used_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_height, tx.max_used_block_height); @@ -768,6 +781,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx) GET_FROM_JSON_OBJECT(val, tx.tx, tx); GET_FROM_JSON_OBJECT(val, tx.blob_size, blob_size); + GET_FROM_JSON_OBJECT(val, tx.weight, weight); GET_FROM_JSON_OBJECT(val, tx.fee, fee); GET_FROM_JSON_OBJECT(val, tx.max_used_block_hash, max_used_block_hash); GET_FROM_JSON_OBJECT(val, tx.max_used_block_height, max_used_block_height); @@ -936,51 +950,70 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp void toJsonValue(rapidjson::Document& doc, const rct::rctSig& sig, rapidjson::Value& val) { + using boost::adaptors::transform; + val.SetObject(); + const auto just_mask = [] (rct::ctkey const& key) -> rct::key const& + { + return key.mask; + }; + INSERT_INTO_JSON_OBJECT(val, doc, type, sig.type); - INSERT_INTO_JSON_OBJECT(val, doc, message, sig.message); - INSERT_INTO_JSON_OBJECT(val, doc, mixRing, sig.mixRing); - INSERT_INTO_JSON_OBJECT(val, doc, pseudoOuts, sig.pseudoOuts); - INSERT_INTO_JSON_OBJECT(val, doc, ecdhInfo, sig.ecdhInfo); - INSERT_INTO_JSON_OBJECT(val, doc, outPk, sig.outPk); - INSERT_INTO_JSON_OBJECT(val, doc, txnFee, sig.txnFee); - INSERT_INTO_JSON_OBJECT(val, doc, p, sig.p); + INSERT_INTO_JSON_OBJECT(val, doc, encrypted, sig.ecdhInfo); + INSERT_INTO_JSON_OBJECT(val, doc, commitments, transform(sig.outPk, just_mask)); + INSERT_INTO_JSON_OBJECT(val, doc, fee, sig.txnFee); + + // prunable + { + rapidjson::Value prunable; + prunable.SetObject(); + + INSERT_INTO_JSON_OBJECT(prunable, doc, range_proofs, sig.p.rangeSigs); + INSERT_INTO_JSON_OBJECT(prunable, doc, bulletproofs, sig.p.bulletproofs); + INSERT_INTO_JSON_OBJECT(prunable, doc, mlsags, sig.p.MGs); + INSERT_INTO_JSON_OBJECT(prunable, doc, pseudo_outs, sig.get_pseudo_outs()); + + val.AddMember("prunable", prunable, doc.GetAllocator()); + } } void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig) { + using boost::adaptors::transform; + if (!val.IsObject()) { throw WRONG_TYPE("json object"); } + std::vector<rct::key> commitments; + GET_FROM_JSON_OBJECT(val, sig.type, type); - GET_FROM_JSON_OBJECT(val, sig.message, message); - GET_FROM_JSON_OBJECT(val, sig.mixRing, mixRing); - GET_FROM_JSON_OBJECT(val, sig.pseudoOuts, pseudoOuts); - GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, ecdhInfo); - GET_FROM_JSON_OBJECT(val, sig.outPk, outPk); - GET_FROM_JSON_OBJECT(val, sig.txnFee, txnFee); - GET_FROM_JSON_OBJECT(val, sig.p, p); -} + GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted); + GET_FROM_JSON_OBJECT(val, commitments, commitments); + GET_FROM_JSON_OBJECT(val, sig.txnFee, fee); -void toJsonValue(rapidjson::Document& doc, const rct::ctkey& key, rapidjson::Value& val) -{ - val.SetObject(); + // prunable + { + OBJECT_HAS_MEMBER_OR_THROW(val, "prunable"); + const auto& prunable = val["prunable"]; - INSERT_INTO_JSON_OBJECT(val, doc, dest, key.dest); - INSERT_INTO_JSON_OBJECT(val, doc, mask, key.mask); -} + rct::keyV pseudo_outs; -void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key) -{ - if (!val.IsObject()) + GET_FROM_JSON_OBJECT(prunable, sig.p.rangeSigs, range_proofs); + GET_FROM_JSON_OBJECT(prunable, sig.p.bulletproofs, bulletproofs); + GET_FROM_JSON_OBJECT(prunable, sig.p.MGs, mlsags); + GET_FROM_JSON_OBJECT(prunable, pseudo_outs, pseudo_outs); + + sig.get_pseudo_outs() = std::move(pseudo_outs); + } + + sig.outPk.reserve(commitments.size()); + for (rct::key const& commitment : commitments) { - throw WRONG_TYPE("json object"); + sig.outPk.push_back({{}, commitment}); } - GET_FROM_JSON_OBJECT(val, key.dest, dest); - GET_FROM_JSON_OBJECT(val, key.mask, mask); } void toJsonValue(rapidjson::Document& doc, const rct::ecdhTuple& tuple, rapidjson::Value& val) @@ -1002,27 +1035,6 @@ void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple) GET_FROM_JSON_OBJECT(val, tuple.amount, amount); } -void toJsonValue(rapidjson::Document& doc, const rct::rctSigPrunable& sig, rapidjson::Value& val) -{ - val.SetObject(); - - INSERT_INTO_JSON_OBJECT(val, doc, rangeSigs, sig.rangeSigs); - INSERT_INTO_JSON_OBJECT(val, doc, bulletproofs, sig.bulletproofs); - INSERT_INTO_JSON_OBJECT(val, doc, MGs, sig.MGs); -} - -void fromJsonValue(const rapidjson::Value& val, rct::rctSigPrunable& sig) -{ - if (!val.IsObject()) - { - throw WRONG_TYPE("json object"); - } - - GET_FROM_JSON_OBJECT(val, sig.rangeSigs, rangeSigs); - GET_FROM_JSON_OBJECT(val, sig.bulletproofs, bulletproofs); - GET_FROM_JSON_OBJECT(val, sig.MGs, MGs); -} - void toJsonValue(rapidjson::Document& doc, const rct::rangeSig& sig, rapidjson::Value& val) { val.SetObject(); @@ -1040,10 +1052,16 @@ void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig) throw WRONG_TYPE("json object"); } + const auto ci = val.FindMember("Ci"); + if (ci == val.MemberEnd()) + { + throw MISSING_KEY("Ci"); + } + GET_FROM_JSON_OBJECT(val, sig.asig, asig); std::vector<rct::key> keyVector; - cryptonote::json::fromJsonValue(val["Ci"], keyVector); + cryptonote::json::fromJsonValue(ci->value, keyVector); if (!(keyVector.size() == 64)) { throw WRONG_TYPE("key64 (rct::key[64])"); @@ -1098,10 +1116,10 @@ void toJsonValue(rapidjson::Document& doc, const rct::boroSig& sig, rapidjson::V val.SetObject(); std::vector<rct::key> keyVector(sig.s0, std::end(sig.s0)); - INSERT_INTO_JSON_OBJECT(val, doc, s0, sig.s0); + INSERT_INTO_JSON_OBJECT(val, doc, s0, keyVector); keyVector.assign(sig.s1, std::end(sig.s1)); - INSERT_INTO_JSON_OBJECT(val, doc, s1, sig.s1); + INSERT_INTO_JSON_OBJECT(val, doc, s1, keyVector); INSERT_INTO_JSON_OBJECT(val, doc, ee, sig.ee); } @@ -1175,9 +1193,13 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, white_peerlist_size, info.white_peerlist_size); INSERT_INTO_JSON_OBJECT(val, doc, grey_peerlist_size, info.grey_peerlist_size); INSERT_INTO_JSON_OBJECT(val, doc, testnet, info.testnet); + INSERT_INTO_JSON_OBJECT(val, doc, nettype, info.nettype); INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); INSERT_INTO_JSON_OBJECT(val, doc, block_size_limit, info.block_size_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_limit, info.block_weight_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_size_median, info.block_size_median); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_median, info.block_weight_median); INSERT_INTO_JSON_OBJECT(val, doc, start_time, info.start_time); } @@ -1200,12 +1222,39 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.white_peerlist_size, white_peerlist_size); GET_FROM_JSON_OBJECT(val, info.grey_peerlist_size, grey_peerlist_size); GET_FROM_JSON_OBJECT(val, info.testnet, testnet); + GET_FROM_JSON_OBJECT(val, info.nettype, nettype); GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); GET_FROM_JSON_OBJECT(val, info.block_size_limit, block_size_limit); + GET_FROM_JSON_OBJECT(val, info.block_weight_limit, block_weight_limit); + GET_FROM_JSON_OBJECT(val, info.block_size_median, block_size_median); + GET_FROM_JSON_OBJECT(val, info.block_weight_median, block_weight_median); GET_FROM_JSON_OBJECT(val, info.start_time, start_time); } +void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::output_distribution& dist, rapidjson::Value& val) +{ + val.SetObject(); + + INSERT_INTO_JSON_OBJECT(val, doc, distribution, dist.data.distribution); + INSERT_INTO_JSON_OBJECT(val, doc, amount, dist.amount); + INSERT_INTO_JSON_OBJECT(val, doc, start_height, dist.data.start_height); + INSERT_INTO_JSON_OBJECT(val, doc, base, dist.data.base); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, dist.data.distribution, distribution); + GET_FROM_JSON_OBJECT(val, dist.amount, amount); + GET_FROM_JSON_OBJECT(val, dist.data.start_height, start_height); + GET_FROM_JSON_OBJECT(val, dist.data.base, base); +} + } // namespace json } // namespace cryptonote diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index de21ace66..b6384ca0d 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -263,15 +263,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp void toJsonValue(rapidjson::Document& doc, const rct::rctSig& i, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& i, rct::rctSig& sig); -void toJsonValue(rapidjson::Document& doc, const rct::ctkey& key, rapidjson::Value& val); -void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key); - void toJsonValue(rapidjson::Document& doc, const rct::ecdhTuple& tuple, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple); -void toJsonValue(rapidjson::Document& doc, const rct::rctSigPrunable& sig, rapidjson::Value& val); -void fromJsonValue(const rapidjson::Value& val, rct::rctSigPrunable& sig); - void toJsonValue(rapidjson::Document& doc, const rct::rangeSig& sig, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig); @@ -287,6 +281,9 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig); void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& info, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info); +void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::output_distribution& dist, rapidjson::Value& val); +void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); + template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Document& doc, const Map& map, rapidjson::Value& val); diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 5fc382a1e..2050e88e2 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -318,7 +318,7 @@ namespace serialization { * \brief self explanatory */ template<class Stream> - bool do_check_stream_state(Stream& s, boost::mpl::bool_<true>) + bool do_check_stream_state(Stream& s, boost::mpl::bool_<true>, bool noeof) { return s.good(); } @@ -329,13 +329,13 @@ namespace serialization { * \detailed Also checks to make sure that the stream is not at EOF */ template<class Stream> - bool do_check_stream_state(Stream& s, boost::mpl::bool_<false>) + bool do_check_stream_state(Stream& s, boost::mpl::bool_<false>, bool noeof) { bool result = false; if (s.good()) { std::ios_base::iostate state = s.rdstate(); - result = EOF == s.peek(); + result = noeof || EOF == s.peek(); s.clear(state); } return result; @@ -347,9 +347,9 @@ namespace serialization { * \brief calls detail::do_check_stream_state for ar */ template<class Archive> - bool check_stream_state(Archive& ar) + bool check_stream_state(Archive& ar, bool noeof = false) { - return detail::do_check_stream_state(ar.stream(), typename Archive::is_saving()); + return detail::do_check_stream_state(ar.stream(), typename Archive::is_saving(), noeof); } /*! \fn serialize @@ -360,6 +360,17 @@ namespace serialization { inline bool serialize(Archive &ar, T &v) { bool r = do_serialize(ar, v); - return r && check_stream_state(ar); + return r && check_stream_state(ar, false); + } + + /*! \fn serialize + * + * \brief serializes \a v into \a ar + */ + template <class Archive, class T> + inline bool serialize_noeof(Archive &ar, T &v) + { + bool r = do_serialize(ar, v); + return r && check_stream_state(ar, true); } } diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index 8bb295931..c31cdebde 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -48,7 +48,6 @@ target_link_libraries(simplewallet cncrypto common mnemonics - epee ${EPEE_READLINE} version ${Boost_CHRONO_LIBRARY} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 397614328..58d4cdced 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -86,31 +86,29 @@ typedef cryptonote::simple_wallet sw; #define EXTENDED_LOGS_FILE "wallet_details.log" -#define DEFAULT_MIX 6 +#define DEFAULT_MIX 10 -#define MIN_RING_SIZE 7 // Used to inform user about min ring size -- does not track actual protocol - -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" +#define MIN_RING_SIZE 11 // Used to inform user about min ring size -- does not track actual protocol #define LOCK_IDLE_SCOPE() \ bool auto_refresh_enabled = m_auto_refresh_enabled.load(std::memory_order_relaxed); \ m_auto_refresh_enabled.store(false, std::memory_order_relaxed); \ /* stop any background refresh, and take over */ \ m_wallet->stop(); \ - m_idle_mutex.lock(); \ - while (m_auto_refresh_refreshing) \ - m_idle_cond.notify_one(); \ - m_idle_mutex.unlock(); \ -/* if (auto_refresh_run)*/ \ - /*m_auto_refresh_thread.join();*/ \ boost::unique_lock<boost::mutex> lock(m_idle_mutex); \ + m_idle_cond.notify_all(); \ epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \ m_auto_refresh_enabled.store(auto_refresh_enabled, std::memory_order_relaxed); \ }) +#define SCOPED_WALLET_UNLOCK() \ + LOCK_IDLE_SCOPE(); \ + boost::optional<tools::password_container> pwd_container = boost::none; \ + if (m_wallet->ask_password() && !m_wallet->watch_only() && !(pwd_container = get_and_verify_password())) { return true; } \ + tools::wallet_keys_unlocker unlocker(*m_wallet, pwd_container); + enum TransferType { - TransferOriginal, - TransferNew, + Transfer, TransferLocked, }; @@ -130,8 +128,6 @@ namespace const command_line::arg_descriptor<bool> arg_restore_deterministic_wallet = {"restore-deterministic-wallet", sw::tr("Recover wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_restore_multisig_wallet = {"restore-multisig-wallet", sw::tr("Recover multisig wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Generate non-deterministic view and spend keys"), false}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false}; - const command_line::arg_descriptor<bool> arg_untrusted_daemon = {"untrusted-daemon", sw::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> arg_allow_mismatched_daemon_version = {"allow-mismatched-daemon-version", sw::tr("Allow communicating with a daemon that uses a different RPC version"), false}; const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; const command_line::arg_descriptor<bool> arg_do_not_relay = {"do-not-relay", sw::tr("The newly created transaction will not be relayed to the monero network"), false}; @@ -141,28 +137,6 @@ namespace const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; -#ifdef WIN32 - // Translate from CP850 to UTF-8; - // std::getline for a Windows console returns a string in CP437 or CP850; as simplewallet, - // like all of Monero, is assumed to work internally with UTF-8 throughout, even on Windows - // (although only implemented partially), a translation to UTF-8 is needed for input. - // - // Note that if a program is started inside the MSYS2 shell somebody already translates - // console input to UTF-8, but it's not clear how one could detect that in order to avoid - // double-translation; this code here thus breaks UTF-8 input within a MSYS2 shell, - // unfortunately. - // - // Note also that input for passwords is NOT translated, to remain compatible with any - // passwords containing special characters that predate this switch to UTF-8 support. - static std::string cp850_to_utf8(const std::string &cp850_str) - { - boost::locale::generator gen; - gen.locale_cache_enabled(true); - std::locale loc = gen("en_US.CP850"); - return boost::locale::conv::to_utf<char>(cp850_str, loc); - } -#endif - std::string input_line(const std::string& prompt) { #ifdef HAVE_READLINE @@ -171,14 +145,33 @@ namespace std::cout << prompt; std::string buf; +#ifdef _WIN32 + buf = tools::input_line_win(); +#else std::getline(std::cin, buf); -#ifdef WIN32 - buf = cp850_to_utf8(buf); #endif return epee::string_tools::trim(buf); } + epee::wipeable_string input_secure_line(const std::string& prompt) + { +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + auto pwd_container = tools::password_container::prompt(false, prompt.c_str(), false); + if (!pwd_container) + { + MERROR("Failed to read secure line"); + return ""; + } + + epee::wipeable_string buf = pwd_container->password(); + + buf.trim(); + return buf; + } + boost::optional<tools::password_container> password_prompter(const char *prompt, bool verify) { #ifdef HAVE_READLINE @@ -187,14 +180,14 @@ namespace auto pwd_container = tools::password_container::prompt(verify, prompt); if (!pwd_container) { - tools::fail_msg_writer() << tr("failed to read wallet password"); + tools::fail_msg_writer() << sw::tr("failed to read wallet password"); } return pwd_container; } boost::optional<tools::password_container> default_password_prompter(bool verify) { - return password_prompter(verify ? tr("Enter a new password for the wallet") : tr("Wallet password"), verify); + return password_prompter(verify ? sw::tr("Enter a new password for the wallet") : sw::tr("Wallet password"), verify); } inline std::string interpret_rpc_response(bool ok, const std::string& status) @@ -272,7 +265,7 @@ namespace } else { - fail_msg_writer() << tr("invalid argument: must be either 0/1, true/false, y/n, yes/no"); + fail_msg_writer() << sw::tr("invalid argument: must be either 0/1, true/false, y/n, yes/no"); return false; } } @@ -328,18 +321,18 @@ namespace std::string dnssec_str; if (dnssec_valid) { - dnssec_str = tr("DNSSEC validation passed"); + dnssec_str = sw::tr("DNSSEC validation passed"); } else { - dnssec_str = tr("WARNING: DNSSEC validation was unsuccessful, this address may not be correct!"); + dnssec_str = sw::tr("WARNING: DNSSEC validation was unsuccessful, this address may not be correct!"); } std::stringstream prompt; - prompt << tr("For URL: ") << url + prompt << sw::tr("For URL: ") << url << ", " << dnssec_str << std::endl - << tr(" Monero Address = ") << addresses[0] + << sw::tr(" Monero Address = ") << addresses[0] << std::endl - << tr("Is this OK? (Y/n) ") + << sw::tr("Is this OK? (Y/n) ") ; // prompt the user for confirmation given the dns query and dnssec status std::string confirm_dns_ok = input_line(prompt.str()); @@ -349,7 +342,7 @@ namespace } if (!command_line::is_yes(confirm_dns_ok)) { - std::cout << tr("you have cancelled the transfer request") << std::endl; + std::cout << sw::tr("you have cancelled the transfer request") << std::endl; return {}; } return addresses[0]; @@ -370,7 +363,7 @@ namespace uint32_t subaddr_index; if(!epee::string_tools::get_xtype_from_string(subaddr_index, subaddr_index_str)) { - fail_msg_writer() << tr("failed to parse index: ") << subaddr_index_str; + fail_msg_writer() << sw::tr("failed to parse index: ") << subaddr_index_str; subaddr_indices.clear(); return false; } @@ -381,21 +374,10 @@ namespace boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str) { - auto pos = str.find(":"); - bool r = pos != std::string::npos; - uint32_t major; - r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos)); - uint32_t minor; - r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1)); - if (r) - { - return std::make_pair(major, minor); - } - else - { - fail_msg_writer() << tr("invalid format for subaddress lookahead; must be <major>:<minor>"); - return {}; - } + auto r = tools::parse_subaddress_lookahead(str); + if (!r) + fail_msg_writer() << sw::tr("invalid format for subaddress lookahead; must be <major>:<minor>"); + return r; } void handle_transfer_exception(const std::exception_ptr &e, bool trusted_daemon) @@ -407,27 +389,27 @@ namespace } catch (const tools::error::daemon_busy&) { - fail_msg_writer() << tr("daemon is busy. Please try again later."); + fail_msg_writer() << sw::tr("daemon is busy. Please try again later."); } catch (const tools::error::no_connection_to_daemon&) { - fail_msg_writer() << tr("no connection to daemon. Please make sure daemon is running."); + fail_msg_writer() << sw::tr("no connection to daemon. Please make sure daemon is running."); } catch (const tools::error::wallet_rpc_error& e) { LOG_ERROR("RPC error: " << e.to_string()); - fail_msg_writer() << tr("RPC error: ") << e.what(); + fail_msg_writer() << sw::tr("RPC error: ") << e.what(); } - catch (const tools::error::get_random_outs_error &e) + catch (const tools::error::get_outs_error &e) { - fail_msg_writer() << tr("failed to get random outputs to mix: ") << e.what(); + fail_msg_writer() << sw::tr("failed to get random outputs to mix: ") << e.what(); } catch (const tools::error::not_enough_unlocked_money& e) { LOG_PRINT_L0(boost::format("not enough money to transfer, available only %s, sent amount %s") % print_money(e.available()) % print_money(e.tx_amount())); - fail_msg_writer() << tr("Not enough money in unlocked balance"); + fail_msg_writer() << sw::tr("Not enough money in unlocked balance"); warn_of_possible_attack = false; } catch (const tools::error::not_enough_money& e) @@ -435,7 +417,7 @@ namespace LOG_PRINT_L0(boost::format("not enough money to transfer, available only %s, sent amount %s") % print_money(e.available()) % print_money(e.tx_amount())); - fail_msg_writer() << tr("Not enough money in unlocked balance"); + fail_msg_writer() << sw::tr("Not enough money in unlocked balance"); warn_of_possible_attack = false; } catch (const tools::error::tx_not_possible& e) @@ -445,30 +427,30 @@ namespace print_money(e.tx_amount() + e.fee()) % print_money(e.tx_amount()) % print_money(e.fee())); - fail_msg_writer() << tr("Failed to find a way to create transactions. This is usually due to dust which is so small it cannot pay for itself in fees, or trying to send more money than the unlocked balance, or not leaving enough for fees"); + fail_msg_writer() << sw::tr("Failed to find a way to create transactions. This is usually due to dust which is so small it cannot pay for itself in fees, or trying to send more money than the unlocked balance, or not leaving enough for fees"); warn_of_possible_attack = false; } catch (const tools::error::not_enough_outs_to_mix& e) { auto writer = fail_msg_writer(); - writer << tr("not enough outputs for specified ring size") << " = " << (e.mixin_count() + 1) << ":"; + writer << sw::tr("not enough outputs for specified ring size") << " = " << (e.mixin_count() + 1) << ":"; for (std::pair<uint64_t, uint64_t> outs_for_amount : e.scanty_outs()) { - writer << "\n" << tr("output amount") << " = " << print_money(outs_for_amount.first) << ", " << tr("found outputs to use") << " = " << outs_for_amount.second; + writer << "\n" << sw::tr("output amount") << " = " << print_money(outs_for_amount.first) << ", " << sw::tr("found outputs to use") << " = " << outs_for_amount.second; } - writer << tr("Please use sweep_unmixable."); + writer << sw::tr("Please use sweep_unmixable."); } catch (const tools::error::tx_not_constructed&) - { - fail_msg_writer() << tr("transaction was not constructed"); + { + fail_msg_writer() << sw::tr("transaction was not constructed"); warn_of_possible_attack = false; } catch (const tools::error::tx_rejected& e) { - fail_msg_writer() << (boost::format(tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); + fail_msg_writer() << (boost::format(sw::tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); std::string reason = e.reason(); if (!reason.empty()) - fail_msg_writer() << tr("Reason: ") << reason; + fail_msg_writer() << sw::tr("Reason: ") << reason; } catch (const tools::error::tx_sum_overflow& e) { @@ -477,38 +459,38 @@ namespace } catch (const tools::error::zero_destination&) { - fail_msg_writer() << tr("one of destinations is zero"); + fail_msg_writer() << sw::tr("one of destinations is zero"); warn_of_possible_attack = false; } catch (const tools::error::tx_too_big& e) { - fail_msg_writer() << tr("failed to find a suitable way to split transactions"); + fail_msg_writer() << sw::tr("failed to find a suitable way to split transactions"); warn_of_possible_attack = false; } catch (const tools::error::transfer_error& e) { LOG_ERROR("unknown transfer error: " << e.to_string()); - fail_msg_writer() << tr("unknown transfer error: ") << e.what(); + fail_msg_writer() << sw::tr("unknown transfer error: ") << e.what(); } catch (const tools::error::multisig_export_needed& e) { LOG_ERROR("Multisig error: " << e.to_string()); - fail_msg_writer() << tr("Multisig error: ") << e.what(); + fail_msg_writer() << sw::tr("Multisig error: ") << e.what(); warn_of_possible_attack = false; } catch (const tools::error::wallet_internal_error& e) { LOG_ERROR("internal error: " << e.to_string()); - fail_msg_writer() << tr("internal error: ") << e.what(); + fail_msg_writer() << sw::tr("internal error: ") << e.what(); } catch (const std::exception& e) { LOG_ERROR("unexpected error: " << e.what()); - fail_msg_writer() << tr("unexpected error: ") << e.what(); + fail_msg_writer() << sw::tr("unexpected error: ") << e.what(); } if (warn_of_possible_attack) - fail_msg_writer() << tr("There was an error, which could mean the node may be trying to get you to retry creating a transaction, and zero in on which outputs you own. Or it could be a bona fide error. It may be prudent to disconnect from this node, and not try to send a tranasction immediately. Alternatively, connect to another node so the original node cannot correlate information."); + fail_msg_writer() << sw::tr("There was an error, which could mean the node may be trying to get you to retry creating a transaction, and zero in on which outputs you own. Or it could be a bona fide error. It may be prudent to disconnect from this node, and not try to send a transaction immediately. Alternatively, connect to another node so the original node cannot correlate information."); } bool check_file_overwrite(const std::string &filename) @@ -518,13 +500,25 @@ namespace { if (boost::ends_with(filename, ".keys")) { - fail_msg_writer() << boost::format(tr("File %s likely stores wallet private keys! Use a different file name.")) % filename; + fail_msg_writer() << boost::format(sw::tr("File %s likely stores wallet private keys! Use a different file name.")) % filename; return false; } - return command_line::is_yes(input_line((boost::format(tr("File %s already exists. Are you sure to overwrite it? (Y/Yes/N/No): ")) % filename).str())); + return command_line::is_yes(input_line((boost::format(sw::tr("File %s already exists. Are you sure to overwrite it? (Y/Yes/N/No): ")) % filename).str())); } return true; } + + void print_secret_key(const crypto::secret_key &k) + { + static constexpr const char hex[] = u8"0123456789abcdef"; + const uint8_t *ptr = (const uint8_t*)k.data; + for (size_t i = 0, sz = sizeof(k); i < sz; ++i) + { + putchar(hex[*ptr >> 4]); + putchar(hex[*ptr & 15]); + ++ptr; + } + } } bool parse_priority(const std::string& arg, uint32_t& priority) @@ -540,6 +534,18 @@ bool parse_priority(const std::string& arg, uint32_t& priority) return false; } +std::string join_priority_strings(const char *delimiter) +{ + std::string s; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) + { + if (!s.empty()) + s += delimiter; + s += allowed_priority_strings[n]; + } + return s; +} + std::string simple_wallet::get_commands_str() { std::stringstream ss; @@ -574,12 +580,15 @@ std::string simple_wallet::get_command_usage(const std::vector<std::string> &arg bool simple_wallet::viewkey(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); // don't log + PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { - std::cout << "secret: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key) << std::endl; + printf("secret: "); + print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key); + putchar('\n'); } std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_view_public_key) << std::endl; @@ -593,12 +602,15 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto fail_msg_writer() << tr("wallet is watch-only and has no spend key"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); // don't log + PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { - std::cout << "secret: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_spend_secret_key) << std::endl; + printf("secret: "); + print_secret_key(m_wallet->get_account().get_keys().m_spend_secret_key); + putchar('\n'); } std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_spend_public_key) << std::endl; @@ -608,7 +620,7 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto bool simple_wallet::print_seed(bool encrypted) { bool success = false; - std::string seed; + epee::wipeable_string seed; bool ready, multisig; if (m_wallet->key_on_device()) @@ -621,7 +633,8 @@ bool simple_wallet::print_seed(bool encrypted) fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); multisig = m_wallet->multisig(&ready); if (multisig) @@ -641,7 +654,7 @@ bool simple_wallet::print_seed(bool encrypted) epee::wipeable_string seed_pass; if (encrypted) { - auto pwd_container = password_prompter(tr("Enter optional seed encryption passphrase, empty to see raw seed"), true); + auto pwd_container = password_prompter(tr("Enter optional seed offset passphrase, empty to see raw seed"), true); if (std::cin.eof() || !pwd_container) return true; seed_pass = pwd_container->password(); @@ -690,22 +703,36 @@ bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = s fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } - if (!m_wallet->is_deterministic()) - { - fail_msg_writer() << tr("wallet is non-deterministic and has no seed"); - return true; - } - - const auto pwd_container = get_and_verify_password(); - if (pwd_container) + + epee::wipeable_string password; { - std::string mnemonic_language = get_mnemonic_language(); - if (mnemonic_language.empty()) + SCOPED_WALLET_UNLOCK(); + + if (!m_wallet->is_deterministic()) + { + fail_msg_writer() << tr("wallet is non-deterministic and has no seed"); return true; + } - m_wallet->set_seed_language(std::move(mnemonic_language)); - m_wallet->rewrite(m_wallet_file, pwd_container->password()); + // we need the password, even if ask-password is unset + if (!pwd_container) + { + pwd_container = get_and_verify_password(); + if (pwd_container == boost::none) + { + fail_msg_writer() << tr("Incorrect password"); + return true; + } + } + password = pwd_container->password(); } + + std::string mnemonic_language = get_mnemonic_language(); + if (mnemonic_language.empty()) + return true; + + m_wallet->set_seed_language(std::move(mnemonic_language)); + m_wallet->rewrite(m_wallet_file, password); return true; } @@ -726,8 +753,7 @@ bool simple_wallet::change_password(const std::vector<std::string> &args) try { - m_wallet->rewrite(m_wallet_file, pwd_container->password()); - m_wallet->store(); + m_wallet->change_password(m_wallet_file, orig_pwd_container->password(), pwd_container->password()); } catch (const tools::error::wallet_logic_error& e) { @@ -758,21 +784,24 @@ bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std: fail_msg_writer() << tr("Cannot connect to daemon"); return true; } - const uint64_t per_kb_fee = m_wallet->get_per_kb_fee(); - const uint64_t typical_size_kb = 13; - message_writer() << (boost::format(tr("Current fee is %s %s per kB")) % print_money(per_kb_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point())).str(); + const bool per_byte = m_wallet->use_fork_rules(HF_VERSION_PER_BYTE_FEE); + const uint64_t base_fee = m_wallet->get_base_fee(); + const char *base = per_byte ? "byte" : "kB"; + const uint64_t typical_size = per_byte ? 2500 : 13; + const uint64_t size_granularity = per_byte ? 1 : 1024; + message_writer() << (boost::format(tr("Current fee is %s %s per %s")) % print_money(base_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % base).str(); std::vector<uint64_t> fees; for (uint32_t priority = 1; priority <= 4; ++priority) { uint64_t mult = m_wallet->get_fee_multiplier(priority); - fees.push_back(per_kb_fee * typical_size_kb * mult); + fees.push_back(base_fee * typical_size * mult); } std::vector<std::pair<uint64_t, uint64_t>> blocks; try { - uint64_t base_size = typical_size_kb * 1024; - blocks = m_wallet->estimate_backlog(base_size, base_size + 1023, fees); + uint64_t base_size = typical_size * size_granularity; + blocks = m_wallet->estimate_backlog(base_size, base_size + size_granularity - 1, fees); } catch (const std::exception &e) { @@ -830,12 +859,7 @@ bool simple_wallet::prepare_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your password is incorrect."); - return true; - } + SCOPED_WALLET_UNLOCK(); std::string multisig_info = m_wallet->get_multisig_info(); success_msg_writer() << multisig_info; @@ -868,13 +892,6 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your original password was incorrect."); - return true; - } - if (args.size() < 2) { fail_msg_writer() << tr("usage: make_multisig <threshold> <multisiginfo1> [<multisiginfo2>...]"); @@ -889,6 +906,13 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) return true; } + const auto orig_pwd_container = get_and_verify_password(); + if(orig_pwd_container == boost::none) + { + fail_msg_writer() << tr("Your original password was incorrect."); + return true; + } + LOCK_IDLE_SCOPE(); try @@ -900,7 +924,7 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) { success_msg_writer() << tr("Another step is needed"); success_msg_writer() << multisig_extra_info; - success_msg_writer() << tr("Send this multisig info to all other participants, then use finalize_multisig <info1> [<info2>...] with others' multisig info"); + success_msg_writer() << tr("Send this multisig info to all other participants, then use exchange_multisig_keys <info1> [<info2>...] with others' multisig info"); return true; } } @@ -930,6 +954,14 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + + const auto pwd_container = get_and_verify_password(); + if(pwd_container == boost::none) + { + fail_msg_writer() << tr("Your original password was incorrect."); + return true; + } + if (!m_wallet->multisig(&ready)) { fail_msg_writer() << tr("This wallet is not multisig"); @@ -941,12 +973,7 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your original password was incorrect."); - return true; - } + LOCK_IDLE_SCOPE(); if (args.size() < 2) { @@ -956,7 +983,7 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) try { - if (!m_wallet->finalize_multisig(orig_pwd_container->password(), args)) + if (!m_wallet->finalize_multisig(pwd_container->password(), args)) { fail_msg_writer() << tr("Failed to finalize multisig"); return true; @@ -971,6 +998,61 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) return true; } +bool simple_wallet::exchange_multisig_keys(const std::vector<std::string> &args) { + bool ready; + if (m_wallet->key_on_device()) + { + fail_msg_writer() << tr("command not supported by HW wallet"); + return true; + } + if (!m_wallet->multisig(&ready)) + { + fail_msg_writer() << tr("This wallet is not multisig"); + return true; + } + if (ready) + { + fail_msg_writer() << tr("This wallet is already finalized"); + return true; + } + + const auto orig_pwd_container = get_and_verify_password(); + if(orig_pwd_container == boost::none) + { + fail_msg_writer() << tr("Your original password was incorrect."); + return true; + } + + if (args.size() < 2) + { + fail_msg_writer() << tr("usage: exchange_multisig_keys <multisiginfo1> [<multisiginfo2>...]"); + return true; + } + + try + { + std::string multisig_extra_info = m_wallet->exchange_multisig_keys(orig_pwd_container->password(), args); + if (!multisig_extra_info.empty()) + { + message_writer() << tr("Another step is needed"); + message_writer() << multisig_extra_info; + message_writer() << tr("Send this multisig info to all other participants, then use exchange_multisig_keys <info1> [<info2>...] with others' multisig info"); + return true; + } else { + uint32_t threshold, total; + m_wallet->multisig(NULL, &threshold, &total); + success_msg_writer() << tr("Multisig wallet has been successfully created. Current wallet type: ") << threshold << "/" << total; + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to perform multisig keys exchange: ") << e.what(); + return true; + } + + return true; +} + bool simple_wallet::export_multisig(const std::vector<std::string> &args) { bool ready; @@ -994,8 +1076,8 @@ bool simple_wallet::export_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_multisig_info <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) - return true; + + SCOPED_WALLET_UNLOCK(); const std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) @@ -1046,8 +1128,8 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: import_multisig_info <filename1> [<filename2>...] - one for each other participant"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) - return true; + + SCOPED_WALLET_UNLOCK(); std::vector<cryptonote::blobdata> info; for (size_t n = 0; n < args.size(); ++n) @@ -1063,11 +1145,11 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) info.push_back(std::move(data)); } - LOCK_IDLE_SCOPE(); - // all read and parsed, actually import try { + m_in_manual_refresh.store(true, std::memory_order_relaxed); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); size_t n_outputs = m_wallet->import_multisig(info); // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -1078,7 +1160,7 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("Failed to import multisig info: ") << e.what(); return true; } - if (is_daemon_trusted()) + if (m_wallet->is_trusted_daemon()) { try { @@ -1125,7 +1207,8 @@ bool simple_wallet::sign_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: sign_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; std::vector<crypto::hash> txids; @@ -1198,7 +1281,8 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: submit_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); if (!try_connect_to_daemon()) return true; @@ -1230,7 +1314,7 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -1265,7 +1349,8 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_raw_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) @@ -1371,9 +1456,116 @@ bool simple_wallet::print_ring(const std::vector<std::string> &args) bool simple_wallet::set_ring(const std::vector<std::string> &args) { crypto::key_image key_image; + + // try filename first + if (args.size() == 1) + { + if (!epee::file_io_utils::is_file_exist(args[0])) + { + fail_msg_writer() << tr("File doesn't exist"); + return true; + } + + char str[4096]; + std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); + if (f) + { + while (!feof(f.get())) + { + if (!fgets(str, sizeof(str), f.get())) + break; + const size_t len = strlen(str); + if (len > 0 && str[len - 1] == '\n') + str[len - 1] = 0; + if (!str[0]) + continue; + char key_image_str[65], type_str[9]; + int read_after_key_image = 0, read = 0; + int fields = sscanf(str, "%64[abcdefABCDEF0123456789] %n%8s %n", key_image_str, &read_after_key_image, type_str, &read); + if (fields != 2) + { + fail_msg_writer() << tr("Invalid ring specification: ") << str; + continue; + } + key_image_str[64] = 0; + type_str[8] = 0; + crypto::key_image key_image; + if (read_after_key_image == 0 || !epee::string_tools::hex_to_pod(key_image_str, key_image)) + { + fail_msg_writer() << tr("Invalid key image: ") << str; + continue; + } + if (read == read_after_key_image+8 || (strcmp(type_str, "absolute") && strcmp(type_str, "relative"))) + { + fail_msg_writer() << tr("Invalid ring type, expected relative or abosolute: ") << str; + continue; + } + bool relative = !strcmp(type_str, "relative"); + if (read < 0 || (size_t)read > strlen(str)) + { + fail_msg_writer() << tr("Error reading line: ") << str; + continue; + } + bool valid = true; + std::vector<uint64_t> ring; + const char *ptr = str + read; + while (*ptr) + { + unsigned long offset; + int elements = sscanf(ptr, "%lu %n", &offset, &read); + if (elements == 0 || read <= 0 || (size_t)read > strlen(str)) + { + fail_msg_writer() << tr("Error reading line: ") << str; + valid = false; + break; + } + ring.push_back(offset); + ptr += read; + } + if (!valid) + continue; + if (ring.empty()) + { + fail_msg_writer() << tr("Invalid ring: ") << str; + continue; + } + if (relative) + { + for (size_t n = 1; n < ring.size(); ++n) + { + if (ring[n] <= 0) + { + fail_msg_writer() << tr("Invalid relative ring: ") << str; + valid = false; + break; + } + } + } + else + { + for (size_t n = 1; n < ring.size(); ++n) + { + if (ring[n] <= ring[n-1]) + { + fail_msg_writer() << tr("Invalid absolute ring: ") << str; + valid = false; + break; + } + } + } + if (!valid) + continue; + if (!m_wallet->set_ring(key_image, ring, relative)) + fail_msg_writer() << tr("Failed to set ring for key image: ") << key_image << ". " << tr("Continuing."); + } + f.reset(); + } + return true; + } + if (args.size() < 3) { - fail_msg_writer() << tr("usage: set_ring <key_image> absolute|relative <index> [<index>...]"); + fail_msg_writer() << tr("usage: set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"); return true; } @@ -1445,23 +1637,23 @@ bool simple_wallet::set_ring(const std::vector<std::string> &args) bool simple_wallet::blackball(const std::vector<std::string> &args) { - crypto::public_key output; + uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets; if (args.size() == 0) { - fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]"); + fail_msg_writer() << tr("usage: mark_output_spent <amount>/<offset> | <filename> [add]"); return true; } try { - if (epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &amount, &offset) == 2) { - m_wallet->blackball_output(output); + m_wallet->blackball_output(std::make_pair(amount, offset)); } else if (epee::file_io_utils::is_file_exist(args[0])) { - std::vector<crypto::public_key> outputs; - char str[65]; + std::vector<std::pair<uint64_t, uint64_t>> outputs; + char str[256]; std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); if (f) @@ -1475,10 +1667,27 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) str[len - 1] = 0; if (!str[0]) continue; - outputs.push_back(crypto::public_key()); - if (!epee::string_tools::hex_to_pod(str, outputs.back())) + if (sscanf(str, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + fail_msg_writer() << tr("First line is not an amount"); + return true; + } + if (sscanf(str, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets--) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(str, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else { - fail_msg_writer() << tr("Invalid public key: ") << str; + fail_msg_writer() << tr("Invalid output: ") << str; return true; } } @@ -1503,13 +1712,13 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) } else { - fail_msg_writer() << tr("Invalid public key, and file doesn't exist"); + fail_msg_writer() << tr("Invalid output key, and file doesn't exist"); return true; } } catch (const std::exception &e) { - fail_msg_writer() << tr("Failed to blackball output: ") << e.what(); + fail_msg_writer() << tr("Failed to mark output spent: ") << e.what(); } return true; @@ -1517,16 +1726,16 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) bool simple_wallet::unblackball(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: unblackball <output_public_key>"); + fail_msg_writer() << tr("usage: mark_output_unspent <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } @@ -1536,7 +1745,7 @@ bool simple_wallet::unblackball(const std::vector<std::string> &args) } catch (const std::exception &e) { - fail_msg_writer() << tr("Failed to unblackball output: ") << e.what(); + fail_msg_writer() << tr("Failed to mark output unspent: ") << e.what(); } return true; @@ -1544,29 +1753,29 @@ bool simple_wallet::unblackball(const std::vector<std::string> &args) bool simple_wallet::blackballed(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: blackballed <output_public_key>"); + fail_msg_writer() << tr("usage: is_output_spent <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } try { if (m_wallet->is_output_blackballed(output)) - message_writer() << tr("Blackballed: ") << output; + message_writer() << tr("Spent: ") << output.first << "/" << output.second; else - message_writer() << tr("not blackballed: ") << output; + message_writer() << tr("Not spent: ") << output.first << "/" << output.second; } catch (const std::exception &e) { - fail_msg_writer() << tr("Failed to unblackball output: ") << e.what(); + fail_msg_writer() << tr("Failed to check whether output is spent: ") << e.what(); } return true; @@ -1592,6 +1801,27 @@ bool simple_wallet::version(const std::vector<std::string> &args) return true; } +bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func) +{ + std::vector<std::string> tx_aux; + + message_writer(console_color_white, false) << tr("Please confirm the transaction on the device"); + + m_wallet->cold_sign_tx(ptx_vector, exported_txs, dsts_info, tx_aux); + + if (accept_func && !accept_func(exported_txs)) + { + MERROR("Transactions rejected by callback"); + return false; + } + + // aux info + m_wallet->cold_tx_aux_import(exported_txs.ptx, tx_aux); + + // import key images + return m_wallet->import_key_images(exported_txs.key_images); +} + bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { const auto pwd_container = get_and_verify_password(); @@ -1660,6 +1890,8 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* if (ring_size != 0 && ring_size != DEFAULT_MIX+1) message_writer() << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); + else if (ring_size == DEFAULT_MIX) + message_writer() << tr("WARNING: from v8, ring size will be fixed and this setting will be ignored."); const auto pwd_container = get_and_verify_password(); if (pwd_container) @@ -1683,12 +1915,12 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - int priority = 0; + uint32_t priority = 0; try { if (strchr(args[1].c_str(), '-')) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4 "); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } if (args[1] == "0") @@ -1697,11 +1929,23 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } else { - priority = boost::lexical_cast<int>(args[1]); - if (priority < 1 || priority > 4) + bool found = false; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); - return true; + if (allowed_priority_strings[n] == args[1]) + { + found = true; + priority = n; + } + } + if (!found) + { + priority = boost::lexical_cast<int>(args[1]); + if (priority < 1 || priority > 4) + { + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); + return true; + } } } @@ -1715,7 +1959,7 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } catch(const boost::bad_lexical_cast &) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } catch(...) @@ -1731,6 +1975,7 @@ bool simple_wallet::set_auto_refresh(const std::vector<std::string> &args/* = st if (pwd_container) { parse_bool_and_use(args[1], [&](bool auto_refresh) { + m_auto_refresh_enabled.store(false, std::memory_order_relaxed); m_wallet->auto_refresh(auto_refresh); m_idle_mutex.lock(); m_auto_refresh_enabled.store(auto_refresh, std::memory_order_relaxed); @@ -1778,10 +2023,29 @@ bool simple_wallet::set_ask_password(const std::vector<std::string> &args/* = st const auto pwd_container = get_and_verify_password(); if (pwd_container) { - parse_bool_and_use(args[1], [&](bool r) { - m_wallet->ask_password(r); - m_wallet->rewrite(m_wallet_file, pwd_container->password()); - }); + tools::wallet2::AskPasswordType ask = tools::wallet2::AskPasswordToDecrypt; + if (args[1] == "never" || args[1] == "0") + ask = tools::wallet2::AskPasswordNever; + else if (args[1] == "action" || args[1] == "1") + ask = tools::wallet2::AskPasswordOnAction; + else if (args[1] == "encrypt" || args[1] == "decrypt" || args[1] == "2") + ask = tools::wallet2::AskPasswordToDecrypt; + else + { + fail_msg_writer() << tr("invalid argument: must be either 0/never, 1/action, or 2/encrypt/decrypt"); + return true; + } + + const tools::wallet2::AskPasswordType cur_ask = m_wallet->ask_password(); + if (!m_wallet->watch_only()) + { + if (cur_ask == tools::wallet2::AskPasswordToDecrypt && ask != tools::wallet2::AskPasswordToDecrypt) + m_wallet->decrypt_keys(pwd_container->password()); + else if (cur_ask != tools::wallet2::AskPasswordToDecrypt && ask == tools::wallet2::AskPasswordToDecrypt) + m_wallet->encrypt_keys(pwd_container->password()); + } + m_wallet->ask_password(ask); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); } return true; } @@ -1997,6 +2261,46 @@ bool simple_wallet::set_segregation_height(const std::vector<std::string> &args/ return true; } +bool simple_wallet::set_ignore_fractional_outputs(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->ignore_fractional_outputs(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + }); + } + return true; +} + +bool simple_wallet::set_device_name(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + if (args.size() == 0){ + fail_msg_writer() << tr("Device name not specified"); + return true; + } + + m_wallet->device_name(args[0]); + bool r = false; + try { + r = m_wallet->reconnect_device(); + if (!r){ + fail_msg_writer() << tr("Device reconnect failed"); + } + + } catch(const std::exception & e){ + MWARNING("Device reconnect failed: " << e.what()); + fail_msg_writer() << tr("Device reconnect failed: ") << e.what(); + } + + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) @@ -2028,7 +2332,7 @@ simple_wallet::simple_wallet() tr("Stop mining in the daemon.")); m_cmd_binder.set_handler("set_daemon", boost::bind(&simple_wallet::set_daemon, this, _1), - tr("set_daemon <host>[:<port>]"), + tr("set_daemon <host>[:<port>] [trusted|untrusted]"), tr("Set another daemon to connect to.")); m_cmd_binder.set_handler("save_bc", boost::bind(&simple_wallet::save_bc, this, _1), @@ -2051,30 +2355,30 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), tr("Show the blockchain height.")); - m_cmd_binder.set_handler("transfer_original", - boost::bind(&simple_wallet::transfer, this, _1), - tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); - m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer_new, this, _1), - tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), + tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), - tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <addr> <amount> <lockblocks> [<payment_id>]"), - tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"), + tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + m_cmd_binder.set_handler("locked_sweep_all", + boost::bind(&simple_wallet::locked_sweep_all, this, _1), + tr("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"), + tr("Send all unlocked balance to an address and lock it for <lockblocks> (max. 1000000). If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. <priority> is the priority of the sweep. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability.")); m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with ring_size 1")); m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), - tr("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id>]"), - tr("Send all unlocked balance to an address. If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used.")); + tr("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id>]"), + tr("Send all unlocked balance to an address. If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. If the parameter \"outputs=<N>\" is specified and N > 0, wallet splits the transaction into N even outputs.")); m_cmd_binder.set_handler("sweep_below", boost::bind(&simple_wallet::sweep_below, this, _1), tr("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id>]"), tr("Send all unlocked outputs below the threshold to an address.")); m_cmd_binder.set_handler("sweep_single", boost::bind(&simple_wallet::sweep_single, this, _1), - tr("sweep_single [<priority>] [<ring_size>] <key_image> <address> [<payment_id>]"), + tr("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id>]"), tr("Send a single output of the given key image to an address without change.")); m_cmd_binder.set_handler("donate", boost::bind(&simple_wallet::donate, this, _1), @@ -2082,8 +2386,8 @@ simple_wallet::simple_wallet() tr("Donate <amount> to the development team (donate.getmonero.org).")); m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::sign_transfer, this, _1), - tr("sign_transfer [export]"), - tr("Sign a transaction from a file.")); + tr("sign_transfer [export_raw]"), + tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::submit_transfer, this, _1), tr("Submit a signed transaction from a file.")); @@ -2147,15 +2451,15 @@ simple_wallet::simple_wallet() "store-tx-info <1|0>\n " " Whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference.\n " "default-ring-size <n>\n " - " Set the default ring size (default and minimum is 5).\n " + " Set the default ring size (obsolete).\n " "auto-refresh <1|0>\n " " Whether to automatically synchronize new blocks from the daemon.\n " "refresh-type <full|optimize-coinbase|no-coinbase|default>\n " " Set the wallet's refresh behaviour.\n " "priority [0|1|2|3|4]\n " - " Set the fee too default/unimportant/normal/elevated/priority.\n " + " Set the fee to default/unimportant/normal/elevated/priority.\n " "confirm-missing-payment-id <1|0>\n " - "ask-password <1|0>\n " + "ask-password <0|1|2 (or never|action|decrypt)>\n " "unit <monero|millinero|micronero|nanonero|piconero>\n " " Set the default monero (sub-)unit.\n " "min-outputs-count [n]\n " @@ -2191,6 +2495,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::get_tx_key, this, _1), tr("get_tx_key <txid>"), tr("Get the transaction key (r) for a given <txid>.")); + m_cmd_binder.set_handler("set_tx_key", + boost::bind(&simple_wallet::set_tx_key, this, _1), + tr("set_tx_key <txid> <tx_key>"), + tr("Set the transaction key (r) for a given <txid> in case the tx was made by some other device or 3rd party wallet.")); m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("check_tx_key <txid> <txkey> <address>"), @@ -2223,15 +2531,23 @@ simple_wallet::simple_wallet() tr("Check a signature proving that the owner of <address> holds at least this much, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("show_transfers", boost::bind(&simple_wallet::show_transfers, this, _1), - tr("show_transfers [in|out|pending|failed|pool] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"), - tr("Show the incoming/outgoing transfers within an optional height range.")); + tr("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"), + // Seemingly broken formatting to compensate for the backslash before the quotes. + tr("Show the incoming/outgoing transfers within an optional height range.\n\n" + "Output format:\n" + "In or Coinbase: Block Number, \"block\"|\"in\", Time, Amount, Transaction Hash, Payment ID, Subaddress Index, \"-\", Note\n" + "Out: Block Number, \"out\", Time, Amount*, Transaction Hash, Payment ID, Fee, Destinations, Input addresses**, \"-\", Note\n" + "Pool: \"pool\", \"in\", Time, Amount, Transaction Hash, Payment Id, Subaddress Index, \"-\", Note, Double Spend Note\n" + "Pending or Failed: \"failed\"|\"pending\", \"out\", Time, Amount*, Transaction Hash, Payment ID, Fee, Input addresses**, \"-\", Note\n\n" + "* Excluding change and fee.\n" + "** Set of address indices used as inputs in this transfer.")); m_cmd_binder.set_handler("unspent_outputs", boost::bind(&simple_wallet::unspent_outputs, this, _1), tr("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"), tr("Show the unspent outputs of a specified address within an optional amount range.")); m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), - tr("Rescan the blockchain from scratch.")); + tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), tr("set_tx_note <txid> [free text note]"), @@ -2269,6 +2585,14 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::import_key_images, this, _1), tr("import_key_images <file>"), tr("Import a signed key images list and verify their spent status.")); + m_cmd_binder.set_handler("hw_key_images_sync", + boost::bind(&simple_wallet::hw_key_images_sync, this, _1), + tr("hw_key_images_sync"), + tr("Synchronizes key images with the hw wallet.")); + m_cmd_binder.set_handler("hw_reconnect", + boost::bind(&simple_wallet::hw_reconnect, this, _1), + tr("hw_reconnect"), + tr("Attempts to reconnect HW wallet.")); m_cmd_binder.set_handler("export_outputs", boost::bind(&simple_wallet::export_outputs, this, _1), tr("export_outputs <file>"), @@ -2299,6 +2623,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::finalize_multisig, this, _1), tr("finalize_multisig <string> [<string>...]"), tr("Turn this wallet into a multisig wallet, extra step for N-1/N wallets")); + m_cmd_binder.set_handler("exchange_multisig_keys", + boost::bind(&simple_wallet::exchange_multisig_keys, this, _1), + tr("exchange_multisig_keys <string> [<string>...]"), + tr("Performs extra multisig keys exchange rounds. Needed for arbitrary M/N multisig wallets")); m_cmd_binder.set_handler("export_multisig_info", boost::bind(&simple_wallet::export_multisig, this, _1), tr("export_multisig_info <filename>"), @@ -2325,24 +2653,24 @@ simple_wallet::simple_wallet() tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)")); m_cmd_binder.set_handler("set_ring", boost::bind(&simple_wallet::set_ring, this, _1), - tr("set_ring <key_image> absolute|relative <index> [<index>...]"), + tr("set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"), tr("Set the ring used for a given key image, so it can be reused in a fork")); m_cmd_binder.set_handler("save_known_rings", boost::bind(&simple_wallet::save_known_rings, this, _1), tr("save_known_rings"), tr("Save known rings to the shared rings database")); - m_cmd_binder.set_handler("blackball", + m_cmd_binder.set_handler("mark_output_spent", boost::bind(&simple_wallet::blackball, this, _1), - tr("blackball <output public key> | <filename> [add]"), - tr("Blackball output(s) so they never get selected as fake outputs in a ring")); - m_cmd_binder.set_handler("unblackball", + tr("mark_output_spent <amount>/<offset> | <filename> [add]"), + tr("Mark output(s) as spent so they never get selected as fake outputs in a ring")); + m_cmd_binder.set_handler("mark_output_unspent", boost::bind(&simple_wallet::unblackball, this, _1), - tr("unblackball <output public key>"), - tr("Unblackballs an output so it may get selected as a fake output in a ring")); - m_cmd_binder.set_handler("blackballed", + tr("mark_output_unspent <amount>/<offset>"), + tr("Marks an output as unspent so it may get selected as a fake output in a ring")); + m_cmd_binder.set_handler("is_output_spent", boost::bind(&simple_wallet::blackballed, this, _1), - tr("blackballed <output public key>"), - tr("Checks whether an output is blackballed")); + tr("is_output_spent <amount>/<offset>"), + tr("Checks whether an output is marked as spent")); m_cmd_binder.set_handler("version", boost::bind(&simple_wallet::version, this, _1), tr("version"), @@ -2360,6 +2688,17 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) std::string seed_language = m_wallet->get_seed_language(); if (m_use_english_language_names) seed_language = crypto::ElectrumWords::get_english_name_for(seed_language); + std::string priority_string = "invalid"; + uint32_t priority = m_wallet->get_default_priority(); + if (priority < allowed_priority_strings.size()) + priority_string = allowed_priority_strings[priority]; + std::string ask_password_string = "invalid"; + switch (m_wallet->ask_password()) + { + case tools::wallet2::AskPasswordNever: ask_password_string = "never"; break; + case tools::wallet2::AskPasswordOnAction: ask_password_string = "action"; break; + case tools::wallet2::AskPasswordToDecrypt: ask_password_string = "decrypt"; break; + } success_msg_writer() << "seed = " << seed_language; success_msg_writer() << "always-confirm-transfers = " << m_wallet->always_confirm_transfers(); success_msg_writer() << "print-ring-members = " << m_wallet->print_ring_members(); @@ -2367,9 +2706,9 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "default-ring-size = " << (m_wallet->default_mixin() ? m_wallet->default_mixin() + 1 : 0); success_msg_writer() << "auto-refresh = " << m_wallet->auto_refresh(); success_msg_writer() << "refresh-type = " << get_refresh_type_name(m_wallet->get_refresh_type()); - success_msg_writer() << "priority = " << m_wallet->get_default_priority(); + success_msg_writer() << "priority = " << priority<< " (" << priority_string << ")"; success_msg_writer() << "confirm-missing-payment-id = " << m_wallet->confirm_missing_payment_id(); - success_msg_writer() << "ask-password = " << m_wallet->ask_password(); + success_msg_writer() << "ask-password = " << m_wallet->ask_password() << " (" << ask_password_string << ")"; success_msg_writer() << "unit = " << cryptonote::get_unit(cryptonote::get_default_decimal_point()); success_msg_writer() << "min-outputs-count = " << m_wallet->get_min_output_count(); success_msg_writer() << "min-outputs-value = " << cryptonote::print_money(m_wallet->get_min_output_value()); @@ -2384,6 +2723,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) const std::pair<size_t, size_t> lookahead = m_wallet->get_subaddress_lookahead(); success_msg_writer() << "subaddress-lookahead = " << lookahead.first << ":" << lookahead.second; success_msg_writer() << "segregation-height = " << m_wallet->segregation_height(); + success_msg_writer() << "ignore-fractional-outputs = " << m_wallet->ignore_fractional_outputs(); + success_msg_writer() << "device_name = " << m_wallet->device_name(); return true; } else @@ -2422,9 +2763,9 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("default-ring-size", set_default_ring_size, tr("integer >= ") << MIN_RING_SIZE); CHECK_SIMPLE_VARIABLE("auto-refresh", set_auto_refresh, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("refresh-type", set_refresh_type, tr("full (slowest, no assumptions); optimize-coinbase (fast, assumes the whole coinbase is paid to a single address); no-coinbase (fastest, assumes we receive no coinbase transaction), default (same as optimize-coinbase)")); - CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4")); + CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4, or one of ") << join_priority_strings(", ")); CHECK_SIMPLE_VARIABLE("confirm-missing-payment-id", set_confirm_missing_payment_id, tr("0 or 1")); - CHECK_SIMPLE_VARIABLE("ask-password", set_ask_password, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("ask-password", set_ask_password, tr("0|1|2 (or never|action|decrypt)")); CHECK_SIMPLE_VARIABLE("unit", set_unit, tr("monero, millinero, micronero, nanonero, piconero")); CHECK_SIMPLE_VARIABLE("min-outputs-count", set_min_output_count, tr("unsigned integer")); CHECK_SIMPLE_VARIABLE("min-outputs-value", set_min_output_value, tr("amount")); @@ -2438,6 +2779,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("key-reuse-mitigation2", set_key_reuse_mitigation2, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("subaddress-lookahead", set_subaddress_lookahead, tr("<major>:<minor>")); CHECK_SIMPLE_VARIABLE("segregation-height", set_segregation_height, tr("unsigned integer")); + CHECK_SIMPLE_VARIABLE("ignore-fractional-outputs", set_ignore_fractional_outputs, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("device-name", set_device_name, tr("<device_name[:device_spec]>")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -2451,8 +2794,24 @@ bool simple_wallet::set_log(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: set_log <log_level_number_0-4> | <categories>"); return true; } - if (!args.empty()) - mlog_set_log(args[0].c_str()); + if(!args.empty()) + { + uint16_t level = 0; + if(epee::string_tools::get_xtype_from_string(level, args[0])) + { + if(4 < level) + { + fail_msg_writer() << tr("wrong number range, use: set_log <log_level_number_0-4> | <categories>"); + return true; + } + mlog_set_log_level(level); + } + else + { + mlog_set_log(args[0].c_str()); + } + } + success_msg_writer() << "New log categories: " << mlog_get_categories(); return true; } @@ -2545,28 +2904,45 @@ bool simple_wallet::ask_wallet_create_if_needed() * \brief Prints the seed with a nice message * \param seed seed to print */ -void simple_wallet::print_seed(std::string seed) +void simple_wallet::print_seed(const epee::wipeable_string &seed) { success_msg_writer(true) << "\n" << tr("NOTE: the following 25 words can be used to recover access to your wallet. " "Write them down and store them somewhere safe and secure. Please do not store them in " "your email or on file storage services outside of your immediate control.\n"); - boost::replace_nth(seed, " ", 15, "\n"); - boost::replace_nth(seed, " ", 7, "\n"); // don't log - std::cout << seed << std::endl; + int space_index = 0; + size_t len = seed.size(); + for (const char *ptr = seed.data(); len--; ++ptr) + { + if (*ptr == ' ') + { + if (space_index == 15 || space_index == 7) + putchar('\n'); + else + putchar(*ptr); + ++space_index; + } + else + putchar(*ptr); + } + putchar('\n'); + fflush(stdout); } //---------------------------------------------------------------------------------------------------- -static bool might_be_partial_seed(std::string words) +static bool might_be_partial_seed(const epee::wipeable_string &words) { - std::vector<std::string> seed; + std::vector<epee::wipeable_string> seed; - boost::algorithm::trim(words); - boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); + words.split(seed); return seed.size() < 24; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::init(const boost::program_options::variables_map& vm) { + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + m_electrum_seed.wipe(); + }); + const bool testnet = tools::wallet2::has_testnet_option(vm); const bool stagenet = tools::wallet2::has_stagenet_option(vm); if (testnet && stagenet) @@ -2576,7 +2952,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; - std::string multisig_keys; + epee::wipeable_string multisig_keys; if (!handle_command_line(vm)) return false; @@ -2618,8 +2994,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { if (m_restore_multisig_wallet) { - const char *prompt = "Specify multisig seed: "; - m_electrum_seed = input_line(prompt); + const char *prompt = "Specify multisig seed"; + m_electrum_seed = input_secure_line(prompt); if (std::cin.eof()) return false; if (m_electrum_seed.empty()) @@ -2633,8 +3009,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) m_electrum_seed = ""; do { - const char *prompt = m_electrum_seed.empty() ? "Specify Electrum seed: " : "Electrum seed continued: "; - std::string electrum_seed = input_line(prompt); + const char *prompt = m_electrum_seed.empty() ? "Specify Electrum seed" : "Electrum seed continued"; + epee::wipeable_string electrum_seed = input_secure_line(prompt); if (std::cin.eof()) return false; if (electrum_seed.empty()) @@ -2642,18 +3018,21 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("specify a recovery parameter with the --electrum-seed=\"words list here\""); return false; } - m_electrum_seed += electrum_seed + " "; + m_electrum_seed += electrum_seed; + m_electrum_seed += ' '; } while (might_be_partial_seed(m_electrum_seed)); } } if (m_restore_multisig_wallet) { - if (!epee::string_tools::parse_hexstr_to_binbuff(m_electrum_seed, multisig_keys)) + const boost::optional<epee::wipeable_string> parsed = m_electrum_seed.parse_hexstr(); + if (!parsed) { fail_msg_writer() << tr("Multisig seed failed verification"); return false; } + multisig_keys = *parsed; } else { @@ -2664,7 +3043,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } } - auto pwd_container = password_prompter(tr("Enter seed encryption passphrase, empty if none"), false); + auto pwd_container = password_prompter(tr("Enter seed offset passphrase, empty if none"), false); if (std::cin.eof() || !pwd_container) return false; epee::wipeable_string seed_pass = pwd_container->password(); @@ -2675,12 +3054,13 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) crypto::secret_key key; crypto::cn_slow_hash(seed_pass.data(), seed_pass.size(), (crypto::hash&)key); sc_reduce32((unsigned char*)key.data); - multisig_keys = m_wallet->decrypt(multisig_keys, key, true); + multisig_keys = m_wallet->decrypt<epee::wipeable_string>(std::string(multisig_keys.data(), multisig_keys.size()), key, true); } else m_recovery_key = cryptonote::decrypt_key(m_recovery_key, seed_pass); } } + epee::wipeable_string password; if (!m_generate_from_view_key.empty()) { m_wallet_file = m_generate_from_view_key; @@ -2705,20 +3085,19 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - std::string viewkey_string = input_line("View key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); if (std::cin.eof()) return false; if (viewkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key viewkey; + if (!viewkey_string.hex_to_pod(unwrap(unwrap(viewkey)))) { fail_msg_writer() << tr("failed to parse view key secret key"); return false; } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); m_wallet_file=m_generate_from_view_key; @@ -2733,27 +3112,29 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - bool r = new_wallet(vm, info.address, boost::none, viewkey); + auto r = new_wallet(vm, info.address, boost::none, viewkey); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } else if (!m_generate_from_spend_key.empty()) { m_wallet_file = m_generate_from_spend_key; // parse spend secret key - std::string spendkey_string = input_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); if (std::cin.eof()) return false; if (spendkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - if (!epee::string_tools::hex_to_pod(spendkey_string, m_recovery_key)) + if (!spendkey_string.hex_to_pod(unwrap(unwrap(m_recovery_key)))) { fail_msg_writer() << tr("failed to parse spend key secret key"); return false; } - bool r = new_wallet(vm, m_recovery_key, true, false, ""); + auto r = new_wallet(vm, m_recovery_key, true, false, ""); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } else if (!m_generate_from_keys.empty()) { @@ -2779,36 +3160,34 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse spend secret key - std::string spendkey_string = input_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); if (std::cin.eof()) return false; if (spendkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata spendkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data) || spendkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key spendkey; + if (!spendkey_string.hex_to_pod(unwrap(unwrap(spendkey)))) { fail_msg_writer() << tr("failed to parse spend key secret key"); return false; } - crypto::secret_key spendkey = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data()); // parse view secret key - std::string viewkey_string = input_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); if (std::cin.eof()) return false; if (viewkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key viewkey; + if(!viewkey_string.hex_to_pod(unwrap(unwrap(viewkey)))) { fail_msg_writer() << tr("failed to parse view key secret key"); return false; } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); m_wallet_file=m_generate_from_keys; @@ -2830,8 +3209,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("view key does not match standard address"); return false; } - bool r = new_wallet(vm, info.address, spendkey, viewkey); + auto r = new_wallet(vm, info.address, spendkey, viewkey); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } // Asks user for all the data required to merge secret keys from multisig wallets into one master wallet, which then gets full control of the multisig wallet. The resulting wallet will be the same as any other regular wallet. @@ -2883,7 +3263,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse secret view key - std::string viewkey_string = input_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); if (std::cin.eof()) return false; if (viewkey_string.empty()) @@ -2891,13 +3271,12 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key viewkey; + if(!viewkey_string.hex_to_pod(unwrap(unwrap(viewkey)))) { fail_msg_writer() << tr("failed to parse secret view key"); return false; } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); // check that the view key matches the given address crypto::public_key pkey; @@ -2918,12 +3297,12 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if(multisig_m == multisig_n) { std::vector<crypto::secret_key> multisig_secret_spendkeys(multisig_n); - std::string spendkey_string; + epee::wipeable_string spendkey_string; cryptonote::blobdata spendkey_data; // get N secret spend keys from user for(unsigned int i=0; i<multisig_n; ++i) { - spendkey_string = input_line(tr((boost::format(tr("Secret spend key (%u of %u):")) % (i+1) % multisig_m).str().c_str())); + spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u):")) % (i+1) % multisig_m).str().c_str())); if (std::cin.eof()) return false; if (spendkey_string.empty()) @@ -2931,12 +3310,11 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data) || spendkey_data.size() != sizeof(crypto::secret_key)) + if(!spendkey_string.hex_to_pod(unwrap(unwrap(multisig_secret_spendkeys[i])))) { fail_msg_writer() << tr("failed to parse spend key secret key"); return false; } - multisig_secret_spendkeys[i] = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data()); } // sum the spend keys together to get the master spend key @@ -2964,16 +3342,19 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // create wallet - bool r = new_wallet(vm, info.address, spendkey, viewkey); + auto r = new_wallet(vm, info.address, spendkey, viewkey); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } else if (!m_generate_from_json.empty()) { - m_wallet_file = m_generate_from_json; try { - m_wallet = tools::wallet2::make_from_json(vm, m_wallet_file, password_prompter); + auto rc = tools::wallet2::make_from_json(vm, false, m_generate_from_json, password_prompter); + m_wallet = std::move(rc.first); + password = rc.second.password(); + m_wallet_file = m_wallet->path(); } catch (const std::exception &e) { @@ -2987,8 +3368,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_device; // create wallet - bool r = new_wallet(vm, "Ledger"); + auto r = new_wallet(vm); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; // if no block_height is specified, assume its a new account and start it "now" if(m_wallet->get_refresh_from_block_height() == 0) { { @@ -3013,12 +3395,13 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } m_wallet_file = m_generate_new; - bool r; + boost::optional<epee::wipeable_string> r; if (m_restore_multisig_wallet) r = new_wallet(vm, multisig_keys, old_language); else r = new_wallet(vm, m_recovery_key, m_restore_deterministic_wallet, m_non_deterministic, old_language); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } if (m_restoring && m_generate_from_json.empty() && m_generate_from_device.empty()) @@ -3100,6 +3483,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } m_wallet->set_refresh_from_block_height(m_restore_height); } + m_wallet->rewrite(m_wallet_file, password); } else { @@ -3118,21 +3502,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - // set --trusted-daemon if local and not overridden - if (!m_trusted_daemon) - { - try - { - if (tools::is_local_address(m_wallet->get_daemon_address())) - { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; - } - } - catch (const std::exception &e) { } - } - - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) message_writer() << (boost::format(tr("Warning: using an untrusted daemon at %s, privacy will be lessened")) % m_wallet->get_daemon_address()).str(); if (m_wallet->get_ring_database().empty()) @@ -3166,10 +3536,6 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet = command_line::get_arg(vm, arg_restore_deterministic_wallet); m_restore_multisig_wallet = command_line::get_arg(vm, arg_restore_multisig_wallet); m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) || !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - m_trusted_daemon = command_line::get_arg(vm, arg_trusted_daemon) && !command_line::get_arg(vm, arg_untrusted_daemon); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) && !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - message_writer() << tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted"); m_allow_mismatched_daemon_version = command_line::get_arg(vm, arg_allow_mismatched_daemon_version); m_restore_height = command_line::get_arg(vm, arg_restore_height); m_do_not_relay = command_line::get_arg(vm, arg_do_not_relay); @@ -3218,14 +3584,16 @@ bool simple_wallet::try_connect_to_daemon(bool silent, uint32_t* version) */ std::string simple_wallet::get_mnemonic_language() { - std::vector<std::string> language_list; + std::vector<std::string> language_list_self, language_list_english; + const std::vector<std::string> &language_list = m_use_english_language_names ? language_list_english : language_list_self; std::string language_choice; int language_number = -1; - crypto::ElectrumWords::get_language_list(language_list, m_use_english_language_names); + crypto::ElectrumWords::get_language_list(language_list_self, false); + crypto::ElectrumWords::get_language_list(language_list_english, true); std::cout << tr("List of available languages for your wallet's seed:") << std::endl; std::cout << tr("If your display freezes, exit blind with ^C, then run again with --use-english-language-names") << std::endl; int ii; - std::vector<std::string>::iterator it; + std::vector<std::string>::const_iterator it; for (it = language_list.begin(), ii = 0; it != language_list.end(); it++, ii++) { std::cout << ii << " : " << *it << std::endl; @@ -3249,7 +3617,7 @@ std::string simple_wallet::get_mnemonic_language() fail_msg_writer() << tr("invalid language choice entered. Please try again.\n"); } } - return language_list[language_number]; + return language_list_self[language_number]; } //---------------------------------------------------------------------------------------------------- boost::optional<tools::password_container> simple_wallet::get_and_verify_password() const @@ -3266,15 +3634,16 @@ boost::optional<tools::password_container> simple_wallet::get_and_verify_passwor return pwd_container; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, const crypto::secret_key& recovery_key, bool recover, bool two_random, const std::string &old_language) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3310,7 +3679,7 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, } mnemonic_language = get_mnemonic_language(); if (mnemonic_language.empty()) - return false; + return {}; } m_wallet->set_seed_language(mnemonic_language); @@ -3323,16 +3692,19 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, recovery_val = m_wallet->generate(m_wallet_file, std::move(rc.second).password(), recovery_key, recover, two_random, create_address_file); message_writer(console_color_white, true) << tr("Generated new wallet: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); - std::cout << tr("View key: ") << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key) << ENDL; + PAUSE_READLINE(); + std::cout << tr("View key: "); + print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key); + putchar('\n'); } catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } // convert rng value to electrum-style word list - std::string electrum_words; + epee::wipeable_string electrum_words; crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language); @@ -3353,19 +3725,20 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, } success_msg_writer() << "**********************************************************************"; - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, const cryptonote::account_public_address& address, const boost::optional<crypto::secret_key>& spendkey, const crypto::secret_key& viewkey) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3395,22 +3768,23 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, - const std::string &device_name) { - auto rc = tools::wallet2::make_new(vm, password_prompter); +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm) +{ + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3422,30 +3796,33 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, if (m_restore_height) m_wallet->set_refresh_from_block_height(m_restore_height); + auto device_desc = tools::wallet2::device_name_option(vm); try { - m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_name); + bool create_address_file = command_line::get_arg(vm, arg_create_address_file); + m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_desc.empty() ? "Ledger" : device_desc, create_address_file); message_writer(console_color_white, true) << tr("Generated new wallet on hw device: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); } catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, - const std::string &multisig_keys, const std::string &old_language) +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, + const epee::wipeable_string &multisig_keys, const std::string &old_language) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3475,7 +3852,7 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, if (!m_wallet->multisig(&ready, &threshold, &total) || !ready) { fail_msg_writer() << tr("failed to generate new mutlisig wallet"); - return false; + return {}; } message_writer(console_color_white, true) << boost::format(tr("Generated new %u/%u multisig wallet: ")) % threshold % total << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); @@ -3483,10 +3860,10 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) @@ -3510,7 +3887,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) epee::wipeable_string password; try { - auto rc = tools::wallet2::make_from_file(vm, m_wallet_file, password_prompter); + auto rc = tools::wallet2::make_from_file(vm, false, m_wallet_file, password_prompter); m_wallet = std::move(rc.first); password = std::move(std::move(rc.second).password()); if (!m_wallet) @@ -3537,7 +3914,12 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) // NOTE: this is_deprecated() refers to the wallet file format before becoming JSON. It does not refer to the "old english" seed words form of "deprecated" used elsewhere. if (m_wallet->is_deprecated()) { - if (m_wallet->is_deterministic()) + bool is_deterministic; + { + SCOPED_WALLET_UNLOCK(); + is_deterministic = m_wallet->is_deterministic(); + } + if (is_deterministic) { message_writer(console_color_green, false) << "\n" << tr("You had been using " "a deprecated version of the wallet. Please proceed to upgrade your wallet.\n"); @@ -3548,7 +3930,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) m_wallet->rewrite(m_wallet_file, password); // Display the seed - std::string seed; + epee::wipeable_string seed; m_wallet->get_seed(seed); print_seed(seed); } @@ -3667,7 +4049,7 @@ bool simple_wallet::save_watch_only(const std::vector<std::string> &args/* = std //---------------------------------------------------------------------------------------------------- bool simple_wallet::start_mining(const std::vector<std::string>& args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -3685,7 +4067,6 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) req.miner_address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); bool ok = true; - size_t max_mining_threads_count = (std::max)(tools::get_max_concurrency(), static_cast<unsigned>(2)); size_t arg_size = args.size(); if(arg_size >= 3) { @@ -3701,7 +4082,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) { uint16_t num = 1; ok = string_tools::get_xtype_from_string(num, args[0]); - ok = ok && (1 <= num && num <= max_mining_threads_count); + ok = ok && 1 <= num; req.threads_count = num; } else @@ -3711,8 +4092,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) if (!ok) { - fail_msg_writer() << tr("invalid arguments. Please use start_mining [<number_of_threads>] [do_bg_mining] [ignore_battery], " - "<number_of_threads> should be from 1 to ") << max_mining_threads_count; + fail_msg_writer() << tr("invalid arguments. Please use start_mining [<number_of_threads>] [do_bg_mining] [ignore_battery]"); return true; } @@ -3771,13 +4151,40 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args) // If no port has been provided, use the default from config if (!match[3].length()) { - int daemon_port = m_wallet->nettype() == cryptonote::TESTNET ? config::testnet::RPC_DEFAULT_PORT : m_wallet->nettype() == cryptonote::STAGENET ? config::stagenet::RPC_DEFAULT_PORT : config::RPC_DEFAULT_PORT; + int daemon_port = get_config(m_wallet->nettype()).RPC_DEFAULT_PORT; daemon_url = match[1] + match[2] + std::string(":") + std::to_string(daemon_port); } else { daemon_url = args[0]; } LOCK_IDLE_SCOPE(); m_wallet->init(daemon_url); + + if (args.size() == 2) + { + if (args[1] == "trusted") + m_wallet->set_trusted_daemon(true); + else if (args[1] == "untrusted") + m_wallet->set_trusted_daemon(false); + else + { + fail_msg_writer() << tr("Expected trusted or untrusted, got ") << args[1] << ": assuming untrusted"; + m_wallet->set_trusted_daemon(false); + } + } + else + { + m_wallet->set_trusted_daemon(false); + try + { + if (tools::is_local_address(m_wallet->get_daemon_address())) + { + MINFO(tr("Daemon is local, assuming trusted")); + m_wallet->set_trusted_daemon(true); + } + } + catch (const std::exception &e) { } + } + success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (m_wallet->is_trusted_daemon() ? tr("trusted") : tr("untrusted")); } else { fail_msg_writer() << tr("This does not seem to be a valid daemon URL."); } @@ -3818,6 +4225,25 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, tr("txid ") << txid << ", " << print_money(amount) << ", " << tr("idx ") << subaddr_index; + + const uint64_t warn_height = m_wallet->nettype() == TESTNET ? 1000000 : m_wallet->nettype() == STAGENET ? 50000 : 1650000; + if (height >= warn_height) + { + std::vector<tx_extra_field> tx_extra_fields; + parse_tx_extra(tx.extra, tx_extra_fields); // failure ok + tx_extra_nonce extra_nonce; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + crypto::hash8 payment_id8 = crypto::null_hash8; + crypto::hash payment_id = crypto::null_hash; + if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + message_writer() << + tr("NOTE: this transaction uses an encrypted payment ID: consider using subaddresses instead"); + else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + message_writer(console_color_red, false) << + tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead"); + } + } if (m_auto_refresh_refreshing) m_cmd_binder.print_prompt(); else @@ -3846,6 +4272,32 @@ void simple_wallet::on_skip_transaction(uint64_t height, const crypto::hash &txi { } //---------------------------------------------------------------------------------------------------- +boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char *reason) +{ + // can't ask for password from a background thread + if (!m_in_manual_refresh.load(std::memory_order_relaxed)) + { + message_writer(console_color_red, false) << boost::format(tr("Password needed (%s) - use the refresh command")) % reason; + m_cmd_binder.print_prompt(); + return boost::none; + } + +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + std::string msg = tr("Enter password"); + if (reason && *reason) + msg += std::string(" (") + reason + ")"; + auto pwd_container = tools::password_container::prompt(false, msg.c_str()); + if (!pwd_container) + { + MERROR("Failed to read password"); + return boost::none; + } + + return pwd_container->password(); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init) { if (!try_connect_to_daemon(is_init)) @@ -3869,7 +4321,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(start_height, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -3941,6 +4393,8 @@ bool simple_wallet::show_balance_unlocked(bool detailed) std::string extra; if (m_wallet->has_multisig_partial_key_images()) extra = tr(" (Some owned outputs have partial key images - import_multisig_info needed)"); + else if (m_wallet->has_unknown_key_images()) + extra += tr(" (Some owned outputs have missing key images - import_key_images needed)"); success_msg_writer() << tr("Currently selected account: [") << m_current_subaddress_account << tr("] ") << m_wallet->get_subaddress_label({m_current_subaddress_account, 0}); const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account]; success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag); @@ -4139,12 +4593,7 @@ uint64_t simple_wallet::get_daemon_blockchain_height(std::string& err) { throw std::runtime_error("simple_wallet null wallet"); } - - COMMAND_RPC_GET_HEIGHT::request req; - COMMAND_RPC_GET_HEIGHT::response res = boost::value_initialized<COMMAND_RPC_GET_HEIGHT::response>(); - bool r = m_wallet->invoke_http_json("/getheight", req, res); - err = interpret_rpc_response(r, res.status); - return res.height; + return m_wallet->get_daemon_blockchain_height(err); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::show_blockchain_height(const std::vector<std::string>& args) @@ -4163,7 +4612,7 @@ bool simple_wallet::show_blockchain_height(const std::vector<std::string>& args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_spent(const std::vector<std::string> &args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -4324,12 +4773,9 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_) { // "transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]" - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; - LOCK_IDLE_SCOPE(); - std::vector<std::string> local_args = args_; std::set<uint32_t> subaddr_indices; @@ -4372,8 +4818,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } - const size_t min_args = (transfer_type == TransferLocked) ? 3 : 2; + const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); @@ -4382,38 +4833,38 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<uint8_t> extra; bool payment_id_seen = false; - bool expect_even = (transfer_type == TransferLocked); - if ((expect_even ? 0 : 1) == local_args.size() % 2) + if (!local_args.empty()) { std::string payment_id_str = local_args.back(); - local_args.pop_back(); - crypto::hash payment_id; - bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); - if(r) + bool r = true; + if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } else { crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) + if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) { std::string extra_nonce; set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; } } if(!r) { - fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + fail_msg_writer() << tr("payment id failed to encode"); return true; } - payment_id_seen = true; } uint64_t locked_blocks = 0; @@ -4436,29 +4887,90 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri local_args.pop_back(); } + vector<cryptonote::address_parse_info> dsts_info; vector<cryptonote::tx_destination_entry> dsts; - for (size_t i = 0; i < local_args.size(); i += 2) + size_t num_subaddresses = 0; + for (size_t i = 0; i < local_args.size(); ) { - cryptonote::address_parse_info info; + dsts_info.emplace_back(); + cryptonote::address_parse_info & info = dsts_info.back(); cryptonote::tx_destination_entry de; - if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter)) + bool r = true; + + // check for a URI + std::string address_uri, payment_id_uri, tx_description, recipient_name, error; + std::vector<std::string> unknown_parameters; + uint64_t amount = 0; + bool has_uri = m_wallet->parse_uri(local_args[i], address_uri, payment_id_uri, amount, tx_description, recipient_name, unknown_parameters, error); + if (has_uri) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), address_uri, oa_prompter); + if (payment_id_uri.size() == 16) + { + if (!tools::wallet2::parse_short_payment_id(payment_id_uri, info.payment_id)) + { + fail_msg_writer() << tr("failed to parse short payment ID from URI"); + return true; + } + info.has_payment_id = true; + } + de.amount = amount; + ++i; + } + else if (i + 1 < local_args.size()) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter); + bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); + if(!ok || 0 == de.amount) + { + fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << + ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); + return true; + } + i += 2; + } + else + { + if (boost::starts_with(local_args[i], "monero:")) + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back() << ": " << error; + else + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back(); + return true; + } + + if (!r) { fail_msg_writer() << tr("failed to parse address"); return true; } de.addr = info.address; de.is_subaddress = info.is_subaddress; + num_subaddresses += info.is_subaddress; - if (info.has_payment_id) + if (info.has_payment_id || !payment_id_uri.empty()) { if (payment_id_seen) { - fail_msg_writer() << tr("a single transaction cannot use more than one payment id: ") << local_args[i]; + fail_msg_writer() << tr("a single transaction cannot use more than one payment id"); return true; } + crypto::hash payment_id; std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + if (info.has_payment_id) + { + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + } + else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) + { + set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + } + else + { + fail_msg_writer() << tr("failed to parse payment id, though it was detected"); + return true; + } bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); if(!r) { @@ -4468,19 +4980,11 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; } - bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); - if(!ok || 0 == de.amount) - { - fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << - ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); - return true; - } - dsts.push_back(de); } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -4493,6 +4997,8 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } } + SCOPED_WALLET_UNLOCK(); + try { // figure out what tx will be necessary @@ -4509,17 +5015,14 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } unlock_block = bc_height + locked_blocks; - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); - break; - case TransferNew: - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); break; default: - LOG_ERROR("Unknown transfer method, using original"); + LOG_ERROR("Unknown transfer method, using default"); /* FALLTHRU */ - case TransferOriginal: - ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); - break; + case Transfer: + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); + break; } if (ptx_vector.empty()) @@ -4675,6 +5178,28 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx"; } } + else if (m_wallet->get_account().get_device().has_tx_cold_sign()) + { + try + { + tools::wallet2::signed_tx_set signed_tx; + if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){ + fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet"); + return true; + } + + commit_or_save(signed_tx.ptx, m_do_not_relay); + } + catch (const std::exception& e) + { + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + } + catch (...) + { + LOG_ERROR("Unknown error"); + fail_msg_writer() << tr("unknown error"); + } + } else if (m_wallet->watch_only()) { bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx"); @@ -4694,7 +5219,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -4707,31 +5232,31 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri //---------------------------------------------------------------------------------------------------- bool simple_wallet::transfer(const std::vector<std::string> &args_) { - return transfer_main(TransferOriginal, args_); + return transfer_main(Transfer, args_); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::transfer_new(const std::vector<std::string> &args_) +bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) { - return transfer_main(TransferNew, args_); + return transfer_main(TransferLocked, args_); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) +bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) { - return transfer_main(TransferLocked, args_); + return sweep_main(0, true, args_); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); + try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(is_daemon_trusted()); + auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(); if (ptx_vector.empty()) { @@ -4800,9 +5325,23 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) commit_or_save(ptx_vector, m_do_not_relay); } } + catch (const tools::error::not_enough_unlocked_money& e) + { + fail_msg_writer() << tr("Not enough money in unlocked balance"); + std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay? (Y/Yes/N/No): ")) % print_money(e.available())).str()); + if (std::cin.eof()) + return true; + if (command_line::is_yes(accepted)) + { + try + { + m_wallet->discard_unmixable_outputs(); + } catch (...) {} + } + } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -4813,16 +5352,19 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &args_) +bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args_) { - // sweep_all [index=<N1>[,<N2>,...]] [<ring_size>] <address> [<payment_id>] + auto print_usage = [below]() + { + fail_msg_writer() << boost::format(tr("usage: %s [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id>]")) % (below ? "sweep_below" : "sweep_all"); + }; if (args_.size() == 0) { fail_msg_writer() << tr("No address given"); + print_usage(); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; @@ -4832,7 +5374,10 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") { if (!parse_subaddress_indices(local_args[0], subaddr_indices)) + { + print_usage(); return true; + } local_args.erase(local_args.begin()); } @@ -4868,10 +5413,69 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } + + uint64_t unlock_block = 0; + if (locked) { + uint64_t locked_blocks = 0; + + if (local_args.size() < 2) { + fail_msg_writer() << tr("missing lockedblocks parameter"); + return true; + } + + try + { + locked_blocks = boost::lexical_cast<uint64_t>(local_args[1]); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("bad locked_blocks parameter"); + return true; + } + if (locked_blocks > 1000000) + { + fail_msg_writer() << tr("Locked blocks too high, max 1000000 (˜4 yrs)"); + return true; + } + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (!err.empty()) + { + fail_msg_writer() << tr("failed to get blockchain height: ") << err; + return true; + } + unlock_block = bc_height + locked_blocks; + + local_args.erase(local_args.begin() + 1); + } + + size_t outputs = 1; + if (local_args.size() > 0 && local_args[0].substr(0, 8) == "outputs=") + { + if (!epee::string_tools::get_xtype_from_string(outputs, local_args[0].substr(8))) + { + fail_msg_writer() << tr("Failed to parse number of outputs"); + return true; + } + else if (outputs < 1) + { + fail_msg_writer() << tr("Amount of outputs should be greater than 0"); + return true; + } + else + { + local_args.erase(local_args.begin()); + } + } std::vector<uint8_t> extra; bool payment_id_seen = false; - if (2 >= local_args.size()) + if (local_args.size() >= 2) { std::string payment_id_str = local_args.back(); @@ -4900,6 +5504,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a if(!r && local_args.size() == 3) { fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + print_usage(); return true; } if (payment_id_seen) @@ -4910,6 +5515,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[0], oa_prompter)) { fail_msg_writer() << tr("failed to parse address"); + print_usage(); return true; } @@ -4933,7 +5539,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -4946,12 +5552,12 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } } - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, outputs, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); if (ptx_vector.empty()) { @@ -5016,6 +5622,31 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx"; } } + else if (m_wallet->get_account().get_device().has_tx_cold_sign()) + { + try + { + tools::wallet2::signed_tx_set signed_tx; + std::vector<cryptonote::address_parse_info> dsts_info; + dsts_info.push_back(info); + + if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){ + fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet"); + return true; + } + + commit_or_save(signed_tx.ptx, m_do_not_relay); + } + catch (const std::exception& e) + { + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + } + catch (...) + { + LOG_ERROR("Unknown error"); + fail_msg_writer() << tr("unknown error"); + } + } else if (m_wallet->watch_only()) { bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx"); @@ -5035,7 +5666,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5048,7 +5679,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_single(const std::vector<std::string> &args_) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); if (!try_connect_to_daemon()) return true; @@ -5069,12 +5700,47 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIX; } + else if (ring_size == 0) + { + fail_msg_writer() << tr("Ring size must not be 0"); + return true; + } else { fake_outs_count = ring_size - 1; local_args.erase(local_args.begin()); } } + uint64_t adjusted_fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); + if (adjusted_fake_outs_count > fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } + + size_t outputs = 1; + if (local_args.size() > 0 && local_args[0].substr(0, 8) == "outputs=") + { + if (!epee::string_tools::get_xtype_from_string(outputs, local_args[0].substr(8))) + { + fail_msg_writer() << tr("Failed to parse number of outputs"); + return true; + } + else if (outputs < 1) + { + fail_msg_writer() << tr("Amount of outputs should be greater than 0"); + return true; + } + else + { + local_args.erase(local_args.begin()); + } + } std::vector<uint8_t> extra; bool payment_id_seen = false; @@ -5109,7 +5775,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (local_args.size() != 2) { - fail_msg_writer() << tr("usage: sweep_single [<priority>] [<ring_size>] <key_image> <address> [<payment_id>]"); + fail_msg_writer() << tr("usage: sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id>]"); return true; } @@ -5146,7 +5812,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } // prompt if there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -5164,7 +5830,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, outputs, fake_outs_count, 0 /* unlock_time */, priority, extra); if (ptx_vector.empty()) { @@ -5234,7 +5900,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5247,7 +5913,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_all(const std::vector<std::string> &args_) { - return sweep_main(0, args_); + return sweep_main(0, false, args_); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_below(const std::vector<std::string> &args_) @@ -5263,7 +5929,7 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_) fail_msg_writer() << tr("invalid amount threshold"); return true; } - return sweep_main(below, std::vector<std::string>(++args_.begin(), args_.end())); + return sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector<std::string> &args_) @@ -5299,8 +5965,8 @@ bool simple_wallet::donate(const std::vector<std::string> &args_) local_args.push_back(amount_str); if (!payment_id_str.empty()) local_args.push_back(payment_id_str); - message_writer() << tr("Donating ") << amount_str << " to The Monero Project (donate.getmonero.org or "<< MONERO_DONATION_ADDR <<")."; - transfer_new(local_args); + message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str(); + transfer(local_args); return true; } //---------------------------------------------------------------------------------------------------- @@ -5469,12 +6135,13 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("This is a watch only wallet"); return true; } - if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export")) + if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export_raw")) { - fail_msg_writer() << tr("usage: sign_transfer [export]"); + fail_msg_writer() << tr("usage: sign_transfer [export_raw]"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); const bool export_raw = args_.size() == 1; std::vector<tools::wallet2::pending_tx> ptx; @@ -5539,7 +6206,7 @@ bool simple_wallet::submit_transfer(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5563,7 +6230,6 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) fail_msg_writer() << tr("usage: get_tx_key <txid>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } crypto::hash txid; if (!epee::string_tools::hex_to_pod(local_args[0], txid)) @@ -5572,7 +6238,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) return true; } - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; @@ -5592,6 +6258,64 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) } } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) +{ + std::vector<std::string> local_args = args_; + + if(local_args.size() != 2) { + fail_msg_writer() << tr("usage: set_tx_key <txid> <tx_key>"); + return true; + } + + crypto::hash txid; + if (!epee::string_tools::hex_to_pod(local_args[0], txid)) + { + fail_msg_writer() << tr("failed to parse txid"); + return true; + } + + crypto::secret_key tx_key; + std::vector<crypto::secret_key> additional_tx_keys; + try + { + if (!epee::string_tools::hex_to_pod(local_args[1].substr(0, 64), tx_key)) + { + fail_msg_writer() << tr("failed to parse tx_key"); + return true; + } + while(true) + { + local_args[1] = local_args[1].substr(64); + if (local_args[1].empty()) + break; + additional_tx_keys.resize(additional_tx_keys.size() + 1); + if (!epee::string_tools::hex_to_pod(local_args[1].substr(0, 64), additional_tx_keys.back())) + { + fail_msg_writer() << tr("failed to parse tx_key"); + return true; + } + } + } + catch (const std::out_of_range &e) + { + fail_msg_writer() << tr("failed to parse tx_key"); + return true; + } + + LOCK_IDLE_SCOPE(); + + try + { + m_wallet->set_tx_key(txid, tx_key, additional_tx_keys); + success_msg_writer() << tr("Tx key successfully stored."); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to store tx key: ") << e.what(); + } + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) { if (m_wallet->key_on_device()) @@ -5619,7 +6343,7 @@ bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); try { @@ -5834,7 +6558,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); try { @@ -5929,9 +6653,7 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } - - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); try { @@ -6016,10 +6738,7 @@ static std::string get_human_readable_timestamp(uint64_t ts) #endif uint64_t now = time(NULL); uint64_t diff = ts > now ? ts - now : now - ts; - if (diff > 24*3600) - strftime(buffer, sizeof(buffer), "%Y-%m-%d", &tm); - else - strftime(buffer, sizeof(buffer), "%I:%M:%S %p", &tm); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); return std::string(buffer); } //---------------------------------------------------------------------------------------------------- @@ -6027,16 +6746,16 @@ static std::string get_human_readable_timespan(std::chrono::seconds seconds) { uint64_t ts = seconds.count(); if (ts < 60) - return std::to_string(ts) + tr(" seconds"); + return std::to_string(ts) + sw::tr(" seconds"); if (ts < 3600) - return std::to_string((uint64_t)(ts / 60)) + tr(" minutes"); + return std::to_string((uint64_t)(ts / 60)) + sw::tr(" minutes"); if (ts < 3600 * 24) - return std::to_string((uint64_t)(ts / 3600)) + tr(" hours"); + return std::to_string((uint64_t)(ts / 3600)) + sw::tr(" hours"); if (ts < 3600 * 24 * 30.5) - return std::to_string((uint64_t)(ts / (3600 * 24))) + tr(" days"); + return std::to_string((uint64_t)(ts / (3600 * 24))) + sw::tr(" days"); if (ts < 3600 * 24 * 365.25) - return std::to_string((uint64_t)(ts / (3600 * 24 * 365.25))) + tr(" months"); - return tr("a long time"); + return std::to_string((uint64_t)(ts / (3600 * 24 * 30.5))) + sw::tr(" months"); + return sw::tr("a long time"); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::show_transfers(const std::vector<std::string> &args_) @@ -6047,12 +6766,13 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) bool pending = true; bool failed = true; bool pool = true; + bool coinbase = true; uint64_t min_height = 0; uint64_t max_height = (uint64_t)-1; boost::optional<uint32_t> subaddr_index; if(local_args.size() > 4) { - fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); + fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); return true; } @@ -6065,19 +6785,24 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) local_args.erase(local_args.begin()); } else if (local_args[0] == "out" || local_args[0] == "outgoing") { - in = pool = false; + in = pool = coinbase = false; local_args.erase(local_args.begin()); } else if (local_args[0] == "pending") { - in = out = failed = false; + in = out = failed = coinbase = false; local_args.erase(local_args.begin()); } else if (local_args[0] == "failed") { - in = out = pending = pool = false; + in = out = pending = pool = coinbase = false; local_args.erase(local_args.begin()); } else if (local_args[0] == "pool") { - in = out = pending = failed = false; + in = out = pending = failed = coinbase = false; + local_args.erase(local_args.begin()); + } + else if (local_args[0] == "coinbase") { + in = out = pending = failed = pool = false; + coinbase = true; local_args.erase(local_args.begin()); } else if (local_args[0] == "all" || local_args[0] == "both") { @@ -6118,20 +6843,23 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) local_args.erase(local_args.begin()); } - std::multimap<uint64_t, std::pair<bool,std::string>> output; + std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>> output; PAUSE_READLINE(); - if (in) { + if (in || coinbase) { std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; m_wallet->get_payments(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); for (std::list<std::pair<crypto::hash, tools::wallet2::payment_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { const tools::wallet2::payment_details &pd = i->second; + if (!pd.m_coinbase && !in) + continue; std::string payment_id = string_tools::pod_to_hex(i->first); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%16.16s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); + const std::string type = pd.m_coinbase ? tr("block") : tr("in"); + output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_green, type, (boost::format("%25.25s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); } } @@ -6164,20 +6892,23 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%16.16s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); + output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_magenta, tr("out"), (boost::format("%25.25s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); } } // print in and out sorted by height - for (std::map<uint64_t, std::pair<bool, std::string>>::const_iterator i = output.begin(); i != output.end(); ++i) { - message_writer(i->second.first ? console_color_green : console_color_magenta, false) << + for (std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>>::const_iterator i = output.begin(); i != output.end(); ++i) { + message_writer(std::get<0>(i->second), false) << boost::format("%8.8llu %6.6s %s") % - ((unsigned long long)i->first) % (i->second.first ? tr("in") : tr("out")) % i->second.second; + ((unsigned long long)i->first) % std::get<1>(i->second) % std::get<2>(i->second); } if (pool) { try { + m_in_manual_refresh.store(true, std::memory_order_relaxed); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); + m_wallet->update_pool_state(); std::list<std::pair<crypto::hash, tools::wallet2::pool_payment_details>> payments; m_wallet->get_unconfirmed_payments(payments, m_current_subaddress_account, subaddr_indices); @@ -6190,7 +6921,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string double_spend_note; if (i->second.m_double_spend_seen) double_spend_note = tr("[Double spend seen on the network: this transaction may or may not end up being mined] "); - message_writer() << (boost::format("%8.8s %6.6s %16.16s %20.20s %s %s %d %s %s%s") % "pool" % "in" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); + message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %d %s %s%s") % "pool" % "in" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); } } catch (const std::exception& e) @@ -6213,7 +6944,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string note = m_wallet->get_tx_note(i->first); bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; if ((failed && is_failed) || (!is_failed && pending)) { - message_writer() << (boost::format("%8.8s %6.6s %16.16s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); + message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); } } } @@ -6231,7 +6962,7 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) auto local_args = args_; std::set<uint32_t> subaddr_indices; - if (local_args.size() > 0 && local_args[0].substr(0, 6) != "index=") + if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") { if (!parse_subaddress_indices(local_args[0], subaddr_indices)) return true; @@ -6361,6 +7092,14 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { + message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); + message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); + std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } return refresh_main(0, true); } //---------------------------------------------------------------------------------------------------- @@ -6380,7 +7119,7 @@ void simple_wallet::wallet_idle_thread() { uint64_t fetched_blocks; if (try_connect_to_daemon(true)) - m_wallet->refresh(0, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks); } catch(...) {} m_auto_refresh_refreshing = false; @@ -6421,12 +7160,6 @@ bool simple_wallet::run() void simple_wallet::stop() { m_cmd_binder.stop_handling(); - - m_idle_run.store(false, std::memory_order_relaxed); - m_wallet->stop(); - // make the background refresh thread quit - boost::unique_lock<boost::mutex> lock(m_idle_mutex); - m_idle_cond.notify_one(); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector<std::string>()*/) @@ -7026,7 +7759,8 @@ bool simple_wallet::sign(const std::vector<std::string> &args) fail_msg_writer() << tr("This wallet is multisig and cannot sign"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -7095,14 +7829,14 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args) fail_msg_writer() << tr("wallet is watch-only and cannot export key images"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; try { - LOCK_IDLE_SCOPE(); if (!m_wallet->export_key_images(filename)) { fail_msg_writer() << tr("failed to save file ") << filename; @@ -7127,7 +7861,7 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -7162,6 +7896,73 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::hw_key_images_sync(const std::vector<std::string> &args) +{ + if (!m_wallet->key_on_device()) + { + fail_msg_writer() << tr("command only supported by HW wallet"); + return true; + } + if (!m_wallet->get_account().get_device().has_ki_cold_sync()) + { + fail_msg_writer() << tr("hw wallet does not support cold KI sync"); + return true; + } + if (!m_wallet->is_trusted_daemon()) + { + fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); + return true; + } + + LOCK_IDLE_SCOPE(); + try + { + message_writer(console_color_white, false) << tr("Please confirm the key image sync on the device"); + + uint64_t spent = 0, unspent = 0; + uint64_t height = m_wallet->cold_key_image_sync(spent, unspent); + if (height > 0) + { + success_msg_writer() << tr("Signed key images imported to height ") << height << ", " + << print_money(spent) << tr(" spent, ") << print_money(unspent) << tr(" unspent"); + } else { + fail_msg_writer() << tr("Failed to import key images"); + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to import key images: ") << e.what(); + return true; + } + + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::hw_reconnect(const std::vector<std::string> &args) +{ + if (!m_wallet->key_on_device()) + { + fail_msg_writer() << tr("command only supported by HW wallet"); + return true; + } + + LOCK_IDLE_SCOPE(); + try + { + bool r = m_wallet->reconnect_device(); + if (!r){ + fail_msg_writer() << tr("Failed to reconnect device"); + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to reconnect device: ") << tr(e.what()); + return true; + } + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::export_outputs(const std::vector<std::string> &args) { if (m_wallet->key_on_device()) @@ -7174,27 +7975,16 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_outputs <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; - LOCK_IDLE_SCOPE(); try { - std::vector<tools::wallet2::transfer_details> outs = m_wallet->export_outputs(); - - std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << outs; - - std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); - const cryptonote::account_public_address &keys = m_wallet->get_account().get_keys().m_account_address; - std::string header; - header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); - header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - std::string ciphertext = m_wallet->encrypt_with_view_secret_key(header + oss.str()); - bool r = epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); + std::string data = m_wallet->export_outputs_to_str(); + bool r = epee::file_io_utils::save_string_to_file(filename, data); if (!r) { fail_msg_writer() << tr("failed to save file ") << filename; @@ -7233,63 +8023,16 @@ bool simple_wallet::import_outputs(const std::vector<std::string> &args) fail_msg_writer() << tr("failed to read file ") << filename; return true; } - const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); - if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) - { - fail_msg_writer() << "Bad output export file magic in " << filename; - return true; - } - - try - { - data = m_wallet->decrypt_with_view_secret_key(std::string(data, magiclen)); - } - catch (const std::exception &e) - { - fail_msg_writer() << "Failed to decrypt " << filename << ": " << e.what(); - return true; - } - - const size_t headerlen = 2 * sizeof(crypto::public_key); - if (data.size() < headerlen) - { - fail_msg_writer() << "Bad data size from file " << filename; - return true; - } - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; - const cryptonote::account_public_address &keys = m_wallet->get_account().get_keys().m_account_address; - if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) - { - fail_msg_writer() << "Outputs from " << filename << " are for a different account"; - return true; - } try { - std::string body(data, headerlen); - std::stringstream iss; - iss << body; - std::vector<tools::wallet2::transfer_details> outputs; - try - { - boost::archive::portable_binary_iarchive ar(iss); - ar >> outputs; - } - catch (...) - { - iss.str(""); - iss << body; - boost::archive::binary_iarchive ar(iss); - ar >> outputs; - } - LOCK_IDLE_SCOPE(); - size_t n_outputs = m_wallet->import_outputs(outputs); + SCOPED_WALLET_UNLOCK(); + size_t n_outputs = m_wallet->import_outputs_from_str(data); success_msg_writer() << boost::lexical_cast<std::string>(n_outputs) << " outputs imported"; } catch (const std::exception &e) { - fail_msg_writer() << "Failed to import outputs: " << e.what(); + fail_msg_writer() << "Failed to import outputs " << filename << ": " << e.what(); return true; } @@ -7331,8 +8074,12 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) if (pd.m_unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { uint64_t bh = std::max(pd.m_unlock_time, pd.m_block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE); + uint64_t last_block_reward = m_wallet->get_last_block_reward(); + uint64_t suggested_threshold = last_block_reward ? (pd.m_amount + last_block_reward - 1) / last_block_reward : 0; if (bh >= last_block_height) success_msg_writer() << "Locked: " << (bh - last_block_height) << " blocks to unlock"; + else if (suggested_threshold > 0) + success_msg_writer() << std::to_string(last_block_height - bh) << " confirmations (" << suggested_threshold << " suggested threshold)"; else success_msg_writer() << std::to_string(last_block_height - bh) << " confirmations"; } @@ -7489,6 +8236,8 @@ void simple_wallet::commit_or_save(std::vector<tools::wallet2::pending_tx>& ptx_ //---------------------------------------------------------------------------------------------------- int main(int argc, char* argv[]) { + TRY_ENTRY(); + #ifdef WIN32 // Activate UTF-8 support for Boost filesystem classes on Windows std::locale::global(boost::locale::generator().generate("")); @@ -7512,8 +8261,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_restore_multisig_wallet ); command_line::add_arg(desc_params, arg_non_deterministic ); command_line::add_arg(desc_params, arg_electrum_seed ); - command_line::add_arg(desc_params, arg_trusted_daemon); - command_line::add_arg(desc_params, arg_untrusted_daemon); command_line::add_arg(desc_params, arg_allow_mismatched_daemon_version); command_line::add_arg(desc_params, arg_restore_height); command_line::add_arg(desc_params, arg_do_not_relay); @@ -7529,7 +8276,7 @@ int main(int argc, char* argv[]) std::tie(vm, should_terminate) = wallet_args::main( argc, argv, "monero-wallet-cli [--wallet-file=<file>|--generate-new-wallet=<file>] [<COMMAND>]", - sw::tr("This is the command line monero wallet. It needs to connect to a monero\ndaemon to work correctly.\nWARNING: Do not reuse your Monero keys on an another fork, UNLESS this fork has key reuse mitigations built in. Doing so will harm your privacy."), + sw::tr("This is the command line monero wallet. It needs to connect to a monero\ndaemon to work correctly.\nWARNING: Do not reuse your Monero keys on another fork, UNLESS this fork has key reuse mitigations built in. Doing so will harm your privacy."), desc_params, positional_options, [](const std::string &s, bool emphasis){ tools::scoped_message_writer(emphasis ? epee::console_color_white : epee::console_color_default, true) << s; }, @@ -7554,7 +8301,7 @@ int main(int argc, char* argv[]) if (!command.empty()) { if (!w.process_command(command)) - fail_msg_writer() << tr("Unknown command: ") << command.front(); + fail_msg_writer() << sw::tr("Unknown command: ") << command.front(); w.stop(); w.deinit(); } @@ -7585,5 +8332,5 @@ int main(int argc, char* argv[]) w.deinit(); } return 0; - //CATCH_ENTRY_L0("main", 1); + CATCH_ENTRY_L0("main", 1); } diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 7a788d432..7d813ceb0 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -44,6 +44,7 @@ #include "cryptonote_basic/cryptonote_basic_impl.h" #include "wallet/wallet2.h" #include "console_handler.h" +#include "wipeable_string.h" #include "common/i18n.h" #include "common/password.h" #include "crypto/crypto.h" // for definition of crypto::secret_key @@ -91,13 +92,13 @@ namespace cryptonote //! \return Prompts user for password and verifies against local file. Logs on error and returns `none` boost::optional<tools::password_container> get_and_verify_password() const; - bool new_wallet(const boost::program_options::variables_map& vm, const crypto::secret_key& recovery_key, + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, const crypto::secret_key& recovery_key, bool recover, bool two_random, const std::string &old_language); - bool new_wallet(const boost::program_options::variables_map& vm, const cryptonote::account_public_address& address, + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, const cryptonote::account_public_address& address, const boost::optional<crypto::secret_key>& spendkey, const crypto::secret_key& viewkey); - bool new_wallet(const boost::program_options::variables_map& vm, - const std::string &multisig_keys, const std::string &old_language); - bool new_wallet(const boost::program_options::variables_map& vm, const std::string& device_name); + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, + const epee::wipeable_string &multisig_keys, const std::string &old_language); + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm); bool open_wallet(const boost::program_options::variables_map& vm); bool close_wallet(); @@ -137,6 +138,8 @@ namespace cryptonote bool set_key_reuse_mitigation2(const std::vector<std::string> &args = std::vector<std::string>()); bool set_subaddress_lookahead(const std::vector<std::string> &args = std::vector<std::string>()); bool set_segregation_height(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_ignore_fractional_outputs(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_device_name(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -150,9 +153,9 @@ namespace cryptonote bool show_blockchain_height(const std::vector<std::string> &args); bool transfer_main(int transfer_type, const std::vector<std::string> &args); bool transfer(const std::vector<std::string> &args); - bool transfer_new(const std::vector<std::string> &args); bool locked_transfer(const std::vector<std::string> &args); - bool sweep_main(uint64_t below, const std::vector<std::string> &args); + bool locked_sweep_all(const std::vector<std::string> &args); + bool sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args); bool sweep_all(const std::vector<std::string> &args); bool sweep_below(const std::vector<std::string> &args); bool sweep_single(const std::vector<std::string> &args); @@ -175,6 +178,7 @@ namespace cryptonote bool rescan_spent(const std::vector<std::string> &args); bool set_log(const std::vector<std::string> &args); bool get_tx_key(const std::vector<std::string> &args); + bool set_tx_key(const std::vector<std::string> &args); bool check_tx_key(const std::vector<std::string> &args); bool get_tx_proof(const std::vector<std::string> &args); bool check_tx_proof(const std::vector<std::string> &args); @@ -197,6 +201,8 @@ namespace cryptonote bool verify(const std::vector<std::string> &args); bool export_key_images(const std::vector<std::string> &args); bool import_key_images(const std::vector<std::string> &args); + bool hw_key_images_sync(const std::vector<std::string> &args); + bool hw_reconnect(const std::vector<std::string> &args); bool export_outputs(const std::vector<std::string> &args); bool import_outputs(const std::vector<std::string> &args); bool show_transfer(const std::vector<std::string> &args); @@ -206,6 +212,7 @@ namespace cryptonote bool prepare_multisig(const std::vector<std::string>& args); bool make_multisig(const std::vector<std::string>& args); bool finalize_multisig(const std::vector<std::string> &args); + bool exchange_multisig_keys(const std::vector<std::string> &args); bool export_multisig(const std::vector<std::string>& args); bool import_multisig(const std::vector<std::string>& args); bool accept_loaded_tx(const tools::wallet2::multisig_tx_set &txs); @@ -219,6 +226,7 @@ namespace cryptonote bool unblackball(const std::vector<std::string>& args); bool blackballed(const std::vector<std::string>& args); bool version(const std::vector<std::string>& args); + bool cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func); uint64_t get_daemon_blockchain_height(std::string& err); bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr); @@ -229,13 +237,12 @@ namespace cryptonote bool print_ring_members(const std::vector<tools::wallet2::pending_tx>& ptx_vector, std::ostream& ostr); std::string get_prompt() const; bool print_seed(bool encrypted); - bool is_daemon_trusted() const { return *m_trusted_daemon; } /*! * \brief Prints the seed with a nice message * \param seed seed to print */ - void print_seed(std::string seed); + void print_seed(const epee::wipeable_string &seed); /*! * \brief Gets the word seed language from the user. @@ -258,6 +265,7 @@ namespace cryptonote virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index); virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); + virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); //---------------------------------------------------------- friend class refresh_progress_reporter_t; @@ -326,13 +334,12 @@ namespace cryptonote std::string m_import_path; std::string m_subaddress_lookahead; - std::string m_electrum_seed; // electrum-style seed parameter + epee::wipeable_string m_electrum_seed; // electrum-style seed parameter crypto::secret_key m_recovery_key; // recovery key (used as random for wallet gen) bool m_restore_deterministic_wallet; // recover flag bool m_restore_multisig_wallet; // recover flag bool m_non_deterministic; // old 2-random generation - boost::optional<bool> m_trusted_daemon; bool m_allow_mismatched_daemon_version; bool m_restoring; // are we restoring, by whatever method? uint64_t m_restore_height; // optional diff --git a/src/version.cpp.in b/src/version.cpp.in index a03da7889..ff2405811 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -1,6 +1,6 @@ #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@" -#define DEF_MONERO_VERSION "0.12.2.0-master" -#define DEF_MONERO_RELEASE_NAME "Lithium Luna" +#define DEF_MONERO_VERSION "0.13.0.0-master" +#define DEF_MONERO_RELEASE_NAME "Beryllium Bullet" #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG #include "version.h" diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index a5a4c7f56..4932dd4b6 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -57,6 +57,7 @@ target_link_libraries(wallet common cryptonote_core mnemonics + device_trezor ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} @@ -85,12 +86,13 @@ monero_add_executable(wallet_rpc_server target_link_libraries(wallet_rpc_server PRIVATE wallet - epee rpc_base cryptonote_core cncrypto common version + daemonizer + ${EPEE_READLINE} ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/wallet/api/address_book.h b/src/wallet/api/address_book.h index 7d9200550..f4ca68efd 100644 --- a/src/wallet/api/address_book.h +++ b/src/wallet/api/address_book.h @@ -42,16 +42,16 @@ public: ~AddressBookImpl(); // Fetches addresses from Wallet2 - void refresh(); - std::vector<AddressBookRow*> getAll() const; - bool addRow(const std::string &dst_addr , const std::string &payment_id, const std::string &description); - bool deleteRow(std::size_t rowId); + void refresh() override; + std::vector<AddressBookRow*> getAll() const override; + bool addRow(const std::string &dst_addr , const std::string &payment_id, const std::string &description) override; + bool deleteRow(std::size_t rowId) override; // Error codes. See AddressBook:ErrorCode enum in wallet2_api.h - std::string errorString() const {return m_errorString;} - int errorCode() const {return m_errorCode;} + std::string errorString() const override {return m_errorString;} + int errorCode() const override {return m_errorCode;} - int lookupPaymentID(const std::string &payment_id) const; + int lookupPaymentID(const std::string &payment_id) const override; private: void clearRows(); diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index 8d200220d..913e3156f 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -200,7 +200,11 @@ std::string PendingTransactionImpl::multisigSignData() { throw std::runtime_error("wallet is not multisig"); } - auto cipher = m_wallet.m_wallet->save_multisig_tx(m_pending_tx); + tools::wallet2::multisig_tx_set txSet; + txSet.m_ptx = m_pending_tx; + txSet.m_signers = m_signers; + auto cipher = m_wallet.m_wallet->save_multisig_tx(txSet); + return epee::string_tools::buff_to_hex_nodelimer(cipher); } catch (const std::exception& e) { m_status = Status_Error; diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index 4f963c134..50b9f07ef 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -43,21 +43,21 @@ class PendingTransactionImpl : public PendingTransaction public: PendingTransactionImpl(WalletImpl &wallet); ~PendingTransactionImpl(); - int status() const; - std::string errorString() const; - bool commit(const std::string &filename = "", bool overwrite = false); - uint64_t amount() const; - uint64_t dust() const; - uint64_t fee() const; - std::vector<std::string> txid() const; - uint64_t txCount() const; - std::vector<uint32_t> subaddrAccount() const; - std::vector<std::set<uint32_t>> subaddrIndices() const; + int status() const override; + std::string errorString() const override; + bool commit(const std::string &filename = "", bool overwrite = false) override; + uint64_t amount() const override; + uint64_t dust() const override; + uint64_t fee() const override; + std::vector<std::string> txid() const override; + uint64_t txCount() const override; + std::vector<uint32_t> subaddrAccount() const override; + std::vector<std::set<uint32_t>> subaddrIndices() const override; // TODO: continue with interface; - std::string multisigSignData(); - void signMultisigTx(); - std::vector<std::string> signersKeys() const; + std::string multisigSignData() override; + void signMultisigTx() override; + std::vector<std::string> signersKeys() const override; private: friend class WalletImpl; diff --git a/src/wallet/api/subaddress.h b/src/wallet/api/subaddress.h index 3f9e37ac8..f3db7d97b 100644 --- a/src/wallet/api/subaddress.h +++ b/src/wallet/api/subaddress.h @@ -40,10 +40,10 @@ public: ~SubaddressImpl(); // Fetches addresses from Wallet2 - void refresh(uint32_t accountIndex); - std::vector<SubaddressRow*> getAll() const; - void addRow(uint32_t accountIndex, const std::string &label); - void setLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label); + void refresh(uint32_t accountIndex) override; + std::vector<SubaddressRow*> getAll() const override; + void addRow(uint32_t accountIndex, const std::string &label) override; + void setLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label) override; private: void clearRows(); diff --git a/src/wallet/api/transaction_info.h b/src/wallet/api/transaction_info.h index 5df9a44ef..37e0445d9 100644 --- a/src/wallet/api/transaction_info.h +++ b/src/wallet/api/transaction_info.h @@ -42,24 +42,24 @@ public: TransactionInfoImpl(); ~TransactionInfoImpl(); //! in/out - virtual int direction() const; + virtual int direction() const override; //! true if hold - virtual bool isPending() const; - virtual bool isFailed() const; - virtual uint64_t amount() const; + virtual bool isPending() const override; + virtual bool isFailed() const override; + virtual uint64_t amount() const override; //! always 0 for incoming txes - virtual uint64_t fee() const; - virtual uint64_t blockHeight() const; - virtual std::set<uint32_t> subaddrIndex() const; - virtual uint32_t subaddrAccount() const; - virtual std::string label() const; + virtual uint64_t fee() const override; + virtual uint64_t blockHeight() const override; + virtual std::set<uint32_t> subaddrIndex() const override; + virtual uint32_t subaddrAccount() const override; + virtual std::string label() const override; - virtual std::string hash() const; - virtual std::time_t timestamp() const; - virtual std::string paymentId() const; - virtual const std::vector<Transfer> &transfers() const; - virtual uint64_t confirmations() const; - virtual uint64_t unlockTime() const; + virtual std::string hash() const override; + virtual std::time_t timestamp() const override; + virtual std::string paymentId() const override; + virtual const std::vector<Transfer> &transfers() const override; + virtual uint64_t confirmations() const override; + virtual uint64_t unlockTime() const override; private: int m_direction; diff --git a/src/wallet/api/unsigned_transaction.h b/src/wallet/api/unsigned_transaction.h index a35630535..8a3330014 100644 --- a/src/wallet/api/unsigned_transaction.h +++ b/src/wallet/api/unsigned_transaction.h @@ -43,19 +43,18 @@ class UnsignedTransactionImpl : public UnsignedTransaction public: UnsignedTransactionImpl(WalletImpl &wallet); ~UnsignedTransactionImpl(); - int status() const; - std::string errorString() const; - std::vector<uint64_t> amount() const; - std::vector<uint64_t> dust() const; - std::vector<uint64_t> fee() const; - std::vector<uint64_t> mixin() const; - std::vector<std::string> paymentId() const; - std::vector<std::string> recipientAddress() const; - uint64_t txCount() const; + int status() const override; + std::string errorString() const override; + std::vector<uint64_t> amount() const override; + std::vector<uint64_t> fee() const override; + std::vector<uint64_t> mixin() const override; + std::vector<std::string> paymentId() const override; + std::vector<std::string> recipientAddress() const override; + uint64_t txCount() const override; // sign txs and save to file - bool sign(const std::string &signedFileName); - std::string confirmationMessage() const {return m_confirmationMessage;} - uint64_t minMixinCount() const; + bool sign(const std::string &signedFileName) override; + std::string confirmationMessage() const override {return m_confirmationMessage;} + uint64_t minMixinCount() const override; private: // Callback function to check all loaded tx's and generate confirmationMessage diff --git a/src/wallet/api/utils.cpp b/src/wallet/api/utils.cpp index aebe41e59..86fe56564 100644 --- a/src/wallet/api/utils.cpp +++ b/src/wallet/api/utils.cpp @@ -51,6 +51,9 @@ bool isAddressLocal(const std::string &address) void onStartup() { tools::on_startup(); +#ifdef NDEBUG + tools::disable_core_dumps(); +#endif } } diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 908e2db2e..ddf2d74ff 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -96,6 +96,9 @@ namespace { throw runtime_error("Multisig wallet is not finalized yet"); } } + void checkMultisigWalletReady(const std::unique_ptr<tools::wallet2> &wallet) { + return checkMultisigWalletReady(wallet.get()); + } void checkMultisigWalletNotReady(const tools::wallet2* wallet) { if (!wallet) { @@ -111,6 +114,9 @@ namespace { throw runtime_error("Multisig wallet is already finalized"); } } + void checkMultisigWalletNotReady(const std::unique_ptr<tools::wallet2> &wallet) { + return checkMultisigWalletNotReady(wallet.get()); + } } struct Wallet2CallbackImpl : public tools::i_wallet2_callback @@ -366,25 +372,26 @@ void Wallet::error(const std::string &category, const std::string &str) { } ///////////////////////// WalletImpl implementation //////////////////////// -WalletImpl::WalletImpl(NetworkType nettype) +WalletImpl::WalletImpl(NetworkType nettype, uint64_t kdf_rounds) :m_wallet(nullptr) , m_status(Wallet::Status_Ok) - , m_trustedDaemon(false) , m_wallet2Callback(nullptr) , m_recoveringFromSeed(false) + , m_recoveringFromDevice(false) , m_synchronized(false) , m_rebuildWalletCache(false) , m_is_connected(false) + , m_refreshShouldRescan(false) { - m_wallet = new tools::wallet2(static_cast<cryptonote::network_type>(nettype)); - m_history = new TransactionHistoryImpl(this); - m_wallet2Callback = new Wallet2CallbackImpl(this); - m_wallet->callback(m_wallet2Callback); + m_wallet.reset(new tools::wallet2(static_cast<cryptonote::network_type>(nettype), kdf_rounds, true)); + m_history.reset(new TransactionHistoryImpl(this)); + m_wallet2Callback.reset(new Wallet2CallbackImpl(this)); + m_wallet->callback(m_wallet2Callback.get()); m_refreshThreadDone = false; m_refreshEnabled = false; - m_addressBook = new AddressBookImpl(this); - m_subaddress = new SubaddressImpl(this); - m_subaddressAccount = new SubaddressAccountImpl(this); + m_addressBook.reset(new AddressBookImpl(this)); + m_subaddress.reset(new SubaddressImpl(this)); + m_subaddressAccount.reset(new SubaddressAccountImpl(this)); m_refreshIntervalMillis = DEFAULT_REFRESH_INTERVAL_MILLIS; @@ -399,18 +406,13 @@ WalletImpl::~WalletImpl() { LOG_PRINT_L1(__FUNCTION__); + m_wallet->callback(NULL); // Pause refresh thread - prevents refresh from starting again pauseRefresh(); // Close wallet - stores cache and stops ongoing refresh operation close(false); // do not store wallet as part of the closing activities // Stop refresh thread stopRefresh(); - delete m_wallet2Callback; - delete m_history; - delete m_addressBook; - delete m_subaddress; - delete m_subaddressAccount; - delete m_wallet; LOG_PRINT_L1(__FUNCTION__ << " finished"); } @@ -419,6 +421,7 @@ bool WalletImpl::create(const std::string &path, const std::string &password, co clearStatus(); m_recoveringFromSeed = false; + m_recoveringFromDevice = false; bool keys_file_exists; bool wallet_file_exists; tools::wallet2::wallet_exists(path, keys_file_exists, wallet_file_exists); @@ -554,18 +557,26 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, } // parse view secret key + bool has_viewkey = true; + crypto::secret_key viewkey; if (viewkey_string.empty()) { - setStatusError(tr("No view key supplied, cancelled")); - return false; + if(has_spendkey) { + has_viewkey = false; + } + else { + setStatusError(tr("Neither view key nor spend key supplied, cancelled")); + return false; + } } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) - { - setStatusError(tr("failed to parse secret view key")); - return false; + if(has_viewkey) { + cryptonote::blobdata viewkey_data; + if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + { + setStatusError(tr("failed to parse secret view key")); + return false; + } + viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); - // check the spend and view keys match the given address crypto::public_key pkey; if(has_spendkey) { @@ -578,26 +589,32 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, return false; } } - if (!crypto::secret_key_to_public_key(viewkey, pkey)) { - setStatusError(tr("failed to verify secret view key")); - return false; - } - if (info.address.m_view_public_key != pkey) { - setStatusError(tr("view key does not match address")); - return false; + if(has_viewkey) { + if (!crypto::secret_key_to_public_key(viewkey, pkey)) { + setStatusError(tr("failed to verify secret view key")); + return false; + } + if (info.address.m_view_public_key != pkey) { + setStatusError(tr("view key does not match address")); + return false; + } } try { - if (has_spendkey) { + if (has_spendkey && has_viewkey) { m_wallet->generate(path, password, info.address, spendkey, viewkey); - setSeedLanguage(language); - LOG_PRINT_L1("Generated new wallet from keys with seed language: " + language); + LOG_PRINT_L1("Generated new wallet from spend key and view key"); } - else { + if(!has_spendkey && has_viewkey) { m_wallet->generate(path, password, info.address, viewkey); LOG_PRINT_L1("Generated new view only wallet from keys"); } + if(has_spendkey && !has_viewkey) { + m_wallet->generate(path, password, spendkey, true, false); + setSeedLanguage(language); + LOG_PRINT_L1("Generated deterministic wallet from spend key with seed language: " + language); + } } catch (const std::exception& e) { @@ -607,11 +624,33 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, return true; } +bool WalletImpl::recoverFromDevice(const std::string &path, const std::string &password, const std::string &device_name) +{ + clearStatus(); + m_recoveringFromSeed = false; + m_recoveringFromDevice = true; + try + { + m_wallet->restore(path, password, device_name); + LOG_PRINT_L1("Generated new wallet from device: " + device_name); + } + catch (const std::exception& e) { + setStatusError(string(tr("failed to generate new wallet: ")) + e.what()); + return false; + } + return true; +} + +Wallet::Device WalletImpl::getDeviceType() const +{ + return static_cast<Wallet::Device>(m_wallet->get_device_type()); +} bool WalletImpl::open(const std::string &path, const std::string &password) { clearStatus(); m_recoveringFromSeed = false; + m_recoveringFromDevice = false; try { // TODO: handle "deprecated" // Check if wallet cache exists @@ -649,6 +688,7 @@ bool WalletImpl::recover(const std::string &path, const std::string &password, c } m_recoveringFromSeed = true; + m_recoveringFromDevice = false; crypto::secret_key recovery_key; std::string old_language; if (!crypto::ElectrumWords::words_to_bytes(seed, recovery_key, old_language)) { @@ -687,6 +727,7 @@ bool WalletImpl::close(bool store) LOG_PRINT_L1("Calling wallet::stop..."); m_wallet->stop(); LOG_PRINT_L1("wallet::stop done"); + m_wallet->deinit(); result = true; clearStatus(); } catch (const std::exception &e) { @@ -698,10 +739,10 @@ bool WalletImpl::close(bool store) std::string WalletImpl::seed() const { - std::string seed; + epee::wipeable_string seed; if (m_wallet) m_wallet->get_seed(seed); - return seed; + return std::string(seed.data(), seed.size()); // TODO } std::string WalletImpl::getSeedLanguage() const @@ -736,7 +777,7 @@ bool WalletImpl::setPassword(const std::string &password) { clearStatus(); try { - m_wallet->rewrite(m_wallet->get_wallet_file(), password); + m_wallet->change_password(m_wallet->get_wallet_file(), m_password, password); m_password = password; } catch (const std::exception &e) { setStatusError(e.what()); @@ -870,6 +911,16 @@ void WalletImpl::setRecoveringFromSeed(bool recoveringFromSeed) m_recoveringFromSeed = recoveringFromSeed; } +void WalletImpl::setRecoveringFromDevice(bool recoveringFromDevice) +{ + m_recoveringFromDevice = recoveringFromDevice; +} + +void WalletImpl::setSubaddressLookahead(uint32_t major, uint32_t minor) +{ + m_wallet->set_subaddress_lookahead(major, minor); +} + uint64_t WalletImpl::balance(uint32_t accountIndex) const { return m_wallet->balance(accountIndex); @@ -961,6 +1012,20 @@ void WalletImpl::refreshAsync() m_refreshCV.notify_one(); } +bool WalletImpl::rescanBlockchain() +{ + clearStatus(); + m_refreshShouldRescan = true; + doRefresh(); + return status() == Status_Ok; +} + +void WalletImpl::rescanBlockchainAsync() +{ + m_refreshShouldRescan = true; + refreshAsync(); +} + void WalletImpl::setAutoRefreshInterval(int millis) { if (millis > MAX_REFRESH_INTERVAL_MILLIS) { @@ -1138,6 +1203,20 @@ string WalletImpl::makeMultisig(const vector<string>& info, uint32_t threshold) return string(); } +std::string WalletImpl::exchangeMultisigKeys(const std::vector<std::string> &info) { + try { + clearStatus(); + checkMultisigWalletNotReady(m_wallet); + + return m_wallet->exchange_multisig_keys(epee::wipeable_string(m_password), info); + } catch (const exception& e) { + LOG_ERROR("Error on exchanging multisig keys: ") << e.what(); + setStatusError(string(tr("Failed to make multisig: ")) + e.what()); + } + + return string(); +} + bool WalletImpl::finalizeMultisig(const vector<string>& extraMultisigInfo) { try { clearStatus(); @@ -1200,6 +1279,20 @@ size_t WalletImpl::importMultisigImages(const vector<string>& images) { return 0; } +bool WalletImpl::hasMultisigPartialKeyImages() const { + try { + clearStatus(); + checkMultisigWalletReady(m_wallet); + + return m_wallet->has_multisig_partial_key_images(); + } catch (const exception& e) { + LOG_ERROR("Error on checking for partial multisig key images: ") << e.what(); + setStatusError(string(tr("Failed to check for partial multisig key images: ")) + e.what()); + } + + return false; +} + PendingTransaction* WalletImpl::restoreMultisigTransaction(const string& signData) { try { clearStatus(); @@ -1253,6 +1346,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIXIN; + fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); uint32_t adjusted_priority = m_wallet->adjust_priority(static_cast<uint32_t>(priority)); @@ -1313,7 +1407,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const dsts.push_back(de); transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } else { // for the GUI, sweep_all (i.e. amount set as "(all)") will always sweep all the funds in all the addresses if (subaddr_indices.empty()) @@ -1321,9 +1415,9 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const for (uint32_t index = 0; index < m_wallet->get_num_subaddresses(subaddr_account); ++index) subaddr_indices.insert(index); } - transaction->m_pending_tx = m_wallet->create_transactions_all(0, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, + transaction->m_pending_tx = m_wallet->create_transactions_all(0, info.address, info.is_subaddress, 1, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } if (multisig().isMultisig) { @@ -1336,8 +1430,8 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error &e) { - setStatusError((boost::format(tr("failed to get random outputs to mix: %s")) % e.what()).str()); + } catch (const tools::error::get_outs_error &e) { + setStatusError((boost::format(tr("failed to get outputs to mix: %s")) % e.what()).str()); } catch (const tools::error::not_enough_unlocked_money& e) { std::ostringstream writer; @@ -1409,7 +1503,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() do { try { - transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(m_trustedDaemon); + transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -1418,8 +1512,8 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error&) { - setStatusError(tr("failed to get random outputs to mix")); + } catch (const tools::error::get_outs_error&) { + setStatusError(tr("failed to get outputs to mix")); } catch (const tools::error::not_enough_unlocked_money& e) { setStatusError(""); std::ostringstream writer; @@ -1487,22 +1581,22 @@ void WalletImpl::disposeTransaction(PendingTransaction *t) TransactionHistory *WalletImpl::history() { - return m_history; + return m_history.get(); } AddressBook *WalletImpl::addressBook() { - return m_addressBook; + return m_addressBook.get(); } Subaddress *WalletImpl::subaddress() { - return m_subaddress; + return m_subaddress.get(); } SubaddressAccount *WalletImpl::subaddressAccount() { - return m_subaddressAccount; + return m_subaddressAccount.get(); } void WalletImpl::setListener(WalletListener *l) @@ -1846,12 +1940,12 @@ Wallet::ConnectionStatus WalletImpl::connected() const void WalletImpl::setTrustedDaemon(bool arg) { - m_trustedDaemon = arg; + m_wallet->set_trusted_daemon(arg); } bool WalletImpl::trustedDaemon() const { - return m_trustedDaemon; + return m_wallet->is_trusted_daemon(); } bool WalletImpl::watchOnly() const @@ -1896,7 +1990,7 @@ void WalletImpl::refreshThreadFunc() // if auto refresh enabled, we wait for the "m_refreshIntervalSeconds" interval. // if not - we wait forever if (m_refreshIntervalMillis > 0) { - boost::posix_time::milliseconds wait_for_ms(m_refreshIntervalMillis); + boost::posix_time::milliseconds wait_for_ms(m_refreshIntervalMillis.load()); m_refreshCV.timed_wait(lock, wait_for_ms); } else { m_refreshCV.wait(lock); @@ -1905,6 +1999,7 @@ void WalletImpl::refreshThreadFunc() LOG_PRINT_L3(__FUNCTION__ << ": refresh lock acquired..."); LOG_PRINT_L3(__FUNCTION__ << ": m_refreshEnabled: " << m_refreshEnabled); LOG_PRINT_L3(__FUNCTION__ << ": m_status: " << status()); + LOG_PRINT_L3(__FUNCTION__ << ": m_refreshShouldRescan: " << m_refreshShouldRescan); if (m_refreshEnabled) { LOG_PRINT_L3(__FUNCTION__ << ": refreshing..."); doRefresh(); @@ -1915,13 +2010,17 @@ void WalletImpl::refreshThreadFunc() void WalletImpl::doRefresh() { + bool rescan = m_refreshShouldRescan.exchange(false); // synchronizing async and sync refresh calls boost::lock_guard<boost::mutex> guarg(m_refreshMutex2); - try { + do try { + LOG_PRINT_L3(__FUNCTION__ << ": doRefresh, rescan = "<<rescan); // Syncing daemon and refreshing wallet simultaneously is very resource intensive. // Disable refresh if wallet is disconnected or daemon isn't synced. if (m_wallet->light_wallet() || daemonSynced()) { - m_wallet->refresh(); + if(rescan) + m_wallet->rescan_blockchain(false); + m_wallet->refresh(trustedDaemon()); if (!m_synchronized) { m_synchronized = true; } @@ -1937,7 +2036,9 @@ void WalletImpl::doRefresh() } } catch (const std::exception &e) { setStatusError(e.what()); - } + break; + }while(!rescan && (rescan=m_refreshShouldRescan.exchange(false))); // repeat if not rescanned and rescan was requested + if (m_wallet2Callback->getListener()) { m_wallet2Callback->getListener()->refreshed(); } @@ -1982,11 +2083,12 @@ bool WalletImpl::isNewWallet() const // with the daemon (pull hashes instead of pull blocks). // If wallet cache is rebuilt, creation height stored in .keys is used. // Watch only wallet is a copy of an existing wallet. - return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_rebuildWalletCache) && !watchOnly(); + return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_recoveringFromDevice || m_rebuildWalletCache) && !watchOnly(); } bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) { + // claim RPC so there's no in-memory encryption for now if (!m_wallet->init(daemon_address, m_daemon_login, upper_transaction_size_limit, ssl)) return false; @@ -2049,41 +2151,83 @@ bool WalletImpl::useForkRules(uint8_t version, int64_t early_blocks) const return m_wallet->use_fork_rules(version,early_blocks); } -bool WalletImpl::blackballOutputs(const std::vector<std::string> &pubkeys, bool add) +bool WalletImpl::blackballOutputs(const std::vector<std::string> &outputs, bool add) { - std::vector<crypto::public_key> raw_pubkeys; - raw_pubkeys.reserve(pubkeys.size()); - for (const std::string &str: pubkeys) + std::vector<std::pair<uint64_t, uint64_t>> raw_outputs; + raw_outputs.reserve(outputs.size()); + uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets; + for (const std::string &str: outputs) { - crypto::public_key pkey; - if (!epee::string_tools::hex_to_pod(str, pkey)) + if (sscanf(str.c_str(), "@%" PRIu64, &amount) == 1) + continue; + if (amount == std::numeric_limits<uint64_t>::max()) { - setStatusError(tr("Failed to parse output public key")); - return false; + setStatusError("First line is not an amount"); + return true; + } + if (sscanf(str.c_str(), "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets--) + raw_outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(str.c_str(), "%" PRIu64, &offset) == 1) + { + raw_outputs.push_back(std::make_pair(amount, offset)); + } + else + { + setStatusError(tr("Invalid output: ") + str); + return false; } - raw_pubkeys.push_back(pkey); } - bool ret = m_wallet->set_blackballed_outputs(raw_pubkeys, add); + bool ret = m_wallet->set_blackballed_outputs(raw_outputs, add); if (!ret) { - setStatusError(tr("Failed to set blackballed outputs")); + setStatusError(tr("Failed to mark outputs as spent")); return false; } return true; } -bool WalletImpl::unblackballOutput(const std::string &pubkey) +bool WalletImpl::blackballOutput(const std::string &amount, const std::string &offset) { - crypto::public_key raw_pubkey; - if (!epee::string_tools::hex_to_pod(pubkey, raw_pubkey)) + uint64_t raw_amount, raw_offset; + if (!epee::string_tools::get_xtype_from_string(raw_amount, amount)) { - setStatusError(tr("Failed to parse output public key")); + setStatusError(tr("Failed to parse output amount")); return false; } - bool ret = m_wallet->unblackball_output(raw_pubkey); + if (!epee::string_tools::get_xtype_from_string(raw_offset, offset)) + { + setStatusError(tr("Failed to parse output offset")); + return false; + } + bool ret = m_wallet->blackball_output(std::make_pair(raw_amount, raw_offset)); if (!ret) { - setStatusError(tr("Failed to unblackball output")); + setStatusError(tr("Failed to mark output as spent")); + return false; + } + return true; +} + +bool WalletImpl::unblackballOutput(const std::string &amount, const std::string &offset) +{ + uint64_t raw_amount, raw_offset; + if (!epee::string_tools::get_xtype_from_string(raw_amount, amount)) + { + setStatusError(tr("Failed to parse output amount")); + return false; + } + if (!epee::string_tools::get_xtype_from_string(raw_offset, offset)) + { + setStatusError(tr("Failed to parse output offset")); + return false; + } + bool ret = m_wallet->unblackball_output(std::make_pair(raw_amount, raw_offset)); + if (!ret) + { + setStatusError(tr("Failed to mark output as unspent")); return false; } return true; @@ -2160,6 +2304,20 @@ void WalletImpl::keyReuseMitigation2(bool mitigation) m_wallet->key_reuse_mitigation2(mitigation); } +bool WalletImpl::lockKeysFile() +{ + return m_wallet->lock_keys_file(); +} + +bool WalletImpl::unlockKeysFile() +{ + return m_wallet->unlock_keys_file(); +} + +bool WalletImpl::isKeysFileLocked() +{ + return m_wallet->is_keys_file_locked(); +} } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 813ca4b30..b4637b8e6 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -52,12 +52,12 @@ struct Wallet2CallbackImpl; class WalletImpl : public Wallet { public: - WalletImpl(NetworkType nettype = MAINNET); + WalletImpl(NetworkType nettype = MAINNET, uint64_t kdf_rounds = 1); ~WalletImpl(); bool create(const std::string &path, const std::string &password, const std::string &language); bool createWatchOnly(const std::string &path, const std::string &password, - const std::string &language) const; + const std::string &language) const override; bool open(const std::string &path, const std::string &password); bool recover(const std::string &path,const std::string &password, const std::string &seed); @@ -76,113 +76,127 @@ public: const std::string &address_string, const std::string &viewkey_string, const std::string &spendkey_string = ""); + bool recoverFromDevice(const std::string &path, + const std::string &password, + const std::string &device_name); + Device getDeviceType() const override; bool close(bool store = true); - std::string seed() const; - std::string getSeedLanguage() const; - void setSeedLanguage(const std::string &arg); + std::string seed() const override; + std::string getSeedLanguage() const override; + void setSeedLanguage(const std::string &arg) override; // void setListener(Listener *) {} - int status() const; - std::string errorString() const; + int status() const override; + std::string errorString() const override; void statusWithErrorString(int& status, std::string& errorString) const override; - bool setPassword(const std::string &password); - std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const; - std::string integratedAddress(const std::string &payment_id) const; - std::string secretViewKey() const; - std::string publicViewKey() const; - std::string secretSpendKey() const; - std::string publicSpendKey() const; - std::string publicMultisigSignerKey() const; - std::string path() const; - bool store(const std::string &path); - std::string filename() const; - std::string keysFilename() const; - bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false); - bool connectToDaemon(); - ConnectionStatus connected() const; - void setTrustedDaemon(bool arg); - bool trustedDaemon() const; - uint64_t balance(uint32_t accountIndex = 0) const; - uint64_t unlockedBalance(uint32_t accountIndex = 0) const; - uint64_t blockChainHeight() const; - uint64_t approximateBlockChainHeight() const; - uint64_t daemonBlockChainHeight() const; - uint64_t daemonBlockChainTargetHeight() const; - bool synchronized() const; - bool refresh(); - void refreshAsync(); - void setAutoRefreshInterval(int millis); - int autoRefreshInterval() const; - void setRefreshFromBlockHeight(uint64_t refresh_from_block_height); - uint64_t getRefreshFromBlockHeight() const { return m_wallet->get_refresh_from_block_height(); }; - void setRecoveringFromSeed(bool recoveringFromSeed); - bool watchOnly() const; - bool rescanSpent(); - NetworkType nettype() const {return static_cast<NetworkType>(m_wallet->nettype());} - void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const; - bool useForkRules(uint8_t version, int64_t early_blocks) const; - - void addSubaddressAccount(const std::string& label); - size_t numSubaddressAccounts() const; - size_t numSubaddresses(uint32_t accountIndex) const; - void addSubaddress(uint32_t accountIndex, const std::string& label); - std::string getSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex) const; - void setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label); + bool setPassword(const std::string &password) override; + std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override; + std::string integratedAddress(const std::string &payment_id) const override; + std::string secretViewKey() const override; + std::string publicViewKey() const override; + std::string secretSpendKey() const override; + std::string publicSpendKey() const override; + std::string publicMultisigSignerKey() const override; + std::string path() const override; + bool store(const std::string &path) override; + std::string filename() const override; + std::string keysFilename() const override; + bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false) override; + bool connectToDaemon() override; + ConnectionStatus connected() const override; + void setTrustedDaemon(bool arg) override; + bool trustedDaemon() const override; + uint64_t balance(uint32_t accountIndex = 0) const override; + uint64_t unlockedBalance(uint32_t accountIndex = 0) const override; + uint64_t blockChainHeight() const override; + uint64_t approximateBlockChainHeight() const override; + uint64_t daemonBlockChainHeight() const override; + uint64_t daemonBlockChainTargetHeight() const override; + bool synchronized() const override; + bool refresh() override; + void refreshAsync() override; + bool rescanBlockchain() override; + void rescanBlockchainAsync() override; + void setAutoRefreshInterval(int millis) override; + int autoRefreshInterval() const override; + void setRefreshFromBlockHeight(uint64_t refresh_from_block_height) override; + uint64_t getRefreshFromBlockHeight() const override { return m_wallet->get_refresh_from_block_height(); }; + void setRecoveringFromSeed(bool recoveringFromSeed) override; + void setRecoveringFromDevice(bool recoveringFromDevice) override; + void setSubaddressLookahead(uint32_t major, uint32_t minor) override; + bool watchOnly() const override; + bool rescanSpent() override; + NetworkType nettype() const override {return static_cast<NetworkType>(m_wallet->nettype());} + void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const override; + bool useForkRules(uint8_t version, int64_t early_blocks) const override; + + void addSubaddressAccount(const std::string& label) override; + size_t numSubaddressAccounts() const override; + size_t numSubaddresses(uint32_t accountIndex) const override; + void addSubaddress(uint32_t accountIndex, const std::string& label) override; + std::string getSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex) const override; + void setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label) override; MultisigState multisig() const override; std::string getMultisigInfo() const override; std::string makeMultisig(const std::vector<std::string>& info, uint32_t threshold) override; + std::string exchangeMultisigKeys(const std::vector<std::string> &info) override; bool finalizeMultisig(const std::vector<std::string>& extraMultisigInfo) override; bool exportMultisigImages(std::string& images) override; size_t importMultisigImages(const std::vector<std::string>& images) override; + bool hasMultisigPartialKeyImages() const override; PendingTransaction* restoreMultisigTransaction(const std::string& signData) override; PendingTransaction * createTransaction(const std::string &dst_addr, const std::string &payment_id, optional<uint64_t> amount, uint32_t mixin_count, PendingTransaction::Priority priority = PendingTransaction::Priority_Low, uint32_t subaddr_account = 0, - std::set<uint32_t> subaddr_indices = {}); - virtual PendingTransaction * createSweepUnmixableTransaction(); - bool submitTransaction(const std::string &fileName); - virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename); - bool exportKeyImages(const std::string &filename); - bool importKeyImages(const std::string &filename); - - virtual void disposeTransaction(PendingTransaction * t); - virtual TransactionHistory * history(); - virtual AddressBook * addressBook(); - virtual Subaddress * subaddress(); - virtual SubaddressAccount * subaddressAccount(); - virtual void setListener(WalletListener * l); - virtual uint32_t defaultMixin() const; - virtual void setDefaultMixin(uint32_t arg); - virtual bool setUserNote(const std::string &txid, const std::string ¬e); - virtual std::string getUserNote(const std::string &txid) const; - virtual std::string getTxKey(const std::string &txid) const; - virtual bool checkTxKey(const std::string &txid, std::string tx_key, const std::string &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); - virtual std::string getTxProof(const std::string &txid, const std::string &address, const std::string &message) const; - virtual bool checkTxProof(const std::string &txid, const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &received, bool &in_pool, uint64_t &confirmations); - virtual std::string getSpendProof(const std::string &txid, const std::string &message) const; - virtual bool checkSpendProof(const std::string &txid, const std::string &message, const std::string &signature, bool &good) const; - virtual std::string getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const; - virtual bool checkReserveProof(const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &total, uint64_t &spent) const; - virtual std::string signMessage(const std::string &message); - virtual bool verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const; - virtual std::string signMultisigParticipant(const std::string &message) const; - virtual bool verifyMessageWithPublicKey(const std::string &message, const std::string &publicKey, const std::string &signature) const; - virtual void startRefresh(); - virtual void pauseRefresh(); - virtual bool parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error); - virtual std::string getDefaultDataDir() const; - virtual bool lightWalletLogin(bool &isNewWallet) const; - virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status); - virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add); - virtual bool unblackballOutput(const std::string &pubkey); - virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const; - virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const; - virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative); - virtual void segregatePreForkOutputs(bool segregate); - virtual void segregationHeight(uint64_t height); - virtual void keyReuseMitigation2(bool mitigation); + std::set<uint32_t> subaddr_indices = {}) override; + virtual PendingTransaction * createSweepUnmixableTransaction() override; + bool submitTransaction(const std::string &fileName) override; + virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename) override; + bool exportKeyImages(const std::string &filename) override; + bool importKeyImages(const std::string &filename) override; + + virtual void disposeTransaction(PendingTransaction * t) override; + virtual TransactionHistory * history() override; + virtual AddressBook * addressBook() override; + virtual Subaddress * subaddress() override; + virtual SubaddressAccount * subaddressAccount() override; + virtual void setListener(WalletListener * l) override; + virtual uint32_t defaultMixin() const override; + virtual void setDefaultMixin(uint32_t arg) override; + virtual bool setUserNote(const std::string &txid, const std::string ¬e) override; + virtual std::string getUserNote(const std::string &txid) const override; + virtual std::string getTxKey(const std::string &txid) const override; + virtual bool checkTxKey(const std::string &txid, std::string tx_key, const std::string &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) override; + virtual std::string getTxProof(const std::string &txid, const std::string &address, const std::string &message) const override; + virtual bool checkTxProof(const std::string &txid, const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &received, bool &in_pool, uint64_t &confirmations) override; + virtual std::string getSpendProof(const std::string &txid, const std::string &message) const override; + virtual bool checkSpendProof(const std::string &txid, const std::string &message, const std::string &signature, bool &good) const override; + virtual std::string getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const override; + virtual bool checkReserveProof(const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &total, uint64_t &spent) const override; + virtual std::string signMessage(const std::string &message) override; + virtual bool verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const override; + virtual std::string signMultisigParticipant(const std::string &message) const override; + virtual bool verifyMessageWithPublicKey(const std::string &message, const std::string &publicKey, const std::string &signature) const override; + virtual void startRefresh() override; + virtual void pauseRefresh() override; + virtual bool parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error) override; + virtual std::string getDefaultDataDir() const override; + virtual bool lightWalletLogin(bool &isNewWallet) const override; + virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status) override; + virtual bool blackballOutputs(const std::vector<std::string> &outputs, bool add) override; + virtual bool blackballOutput(const std::string &amount, const std::string &offset) override; + virtual bool unblackballOutput(const std::string &amount, const std::string &offset) override; + virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const override; + virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const override; + virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) override; + virtual void segregatePreForkOutputs(bool segregate) override; + virtual void segregationHeight(uint64_t height) override; + virtual void keyReuseMitigation2(bool mitigation) override; + virtual bool lockKeysFile() override; + virtual bool unlockKeysFile() override; + virtual bool isKeysFileLocked() override; private: void clearStatus() const; @@ -205,22 +219,22 @@ private: friend class SubaddressImpl; friend class SubaddressAccountImpl; - tools::wallet2 * m_wallet; + std::unique_ptr<tools::wallet2> m_wallet; mutable boost::mutex m_statusMutex; mutable int m_status; mutable std::string m_errorString; std::string m_password; - TransactionHistoryImpl * m_history; - bool m_trustedDaemon; - Wallet2CallbackImpl * m_wallet2Callback; - AddressBookImpl * m_addressBook; - SubaddressImpl * m_subaddress; - SubaddressAccountImpl * m_subaddressAccount; + std::unique_ptr<TransactionHistoryImpl> m_history; + std::unique_ptr<Wallet2CallbackImpl> m_wallet2Callback; + std::unique_ptr<AddressBookImpl> m_addressBook; + std::unique_ptr<SubaddressImpl> m_subaddress; + std::unique_ptr<SubaddressAccountImpl> m_subaddressAccount; // multi-threaded refresh stuff std::atomic<bool> m_refreshEnabled; std::atomic<bool> m_refreshThreadDone; std::atomic<int> m_refreshIntervalMillis; + std::atomic<bool> m_refreshShouldRescan; // synchronizing refresh loop; boost::mutex m_refreshMutex; @@ -232,6 +246,7 @@ private: // so it shouldn't be considered as new and pull blocks (slow-refresh) // instead of pulling hashes (fast-refresh) std::atomic<bool> m_recoveringFromSeed; + std::atomic<bool> m_recoveringFromDevice; std::atomic<bool> m_synchronized; std::atomic<bool> m_rebuildWalletCache; // cache connection status to avoid unnecessary RPC calls diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 5b99bd975..82627de29 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -373,6 +373,10 @@ struct WalletListener */ struct Wallet { + enum Device { + Device_Software = 0, + Device_Ledger = 1 + }; enum Status { Status_Ok, @@ -509,6 +513,21 @@ struct Wallet */ virtual void setRecoveringFromSeed(bool recoveringFromSeed) = 0; + /*! + * \brief setRecoveringFromDevice - set state to recovering from device + * + * \param recoveringFromDevice - true/false + */ + virtual void setRecoveringFromDevice(bool recoveringFromDevice) = 0; + + /*! + * \brief setSubaddressLookahead - set size of subaddress lookahead + * + * \param major - size fot the major index + * \param minor - size fot the minor index + */ + virtual void setSubaddressLookahead(uint32_t major, uint32_t minor) = 0; + /** * @brief connectToDaemon - connects to the daemon. TODO: check if it can be removed * @return @@ -625,6 +644,17 @@ struct Wallet virtual void refreshAsync() = 0; /** + * @brief rescanBlockchain - rescans the wallet, updating transactions from daemon + * @return - true if refreshed successfully; + */ + virtual bool rescanBlockchain() = 0; + + /** + * @brief rescanBlockchainAsync - rescans wallet asynchronously, starting from genesys + */ + virtual void rescanBlockchainAsync() = 0; + + /** * @brief setAutoRefreshInterval - setup interval for automatic refresh. * @param seconds - interval in millis. if zero or less than zero - automatic refresh disabled; */ @@ -688,6 +718,12 @@ struct Wallet */ virtual std::string makeMultisig(const std::vector<std::string>& info, uint32_t threshold) = 0; /** + * @brief exchange_multisig_keys - provides additional key exchange round for arbitrary multisig schemes (like N-1/N, M/N) + * @param info - base58 encoded key derivations returned by makeMultisig or exchangeMultisigKeys function call + * @return new info string if more rounds required or an empty string if wallet creation is done + */ + virtual std::string exchangeMultisigKeys(const std::vector<std::string> &info) = 0; + /** * @brief finalizeMultisig - finalizes N - 1 / N multisig wallets creation * @param extraMultisigInfo - wallet participants' extra multisig info obtained with makeMultisig call * @return true if success @@ -705,6 +741,11 @@ struct Wallet * @return number of imported images */ virtual size_t importMultisigImages(const std::vector<std::string>& images) = 0; + /** + * @brief hasMultisigPartialKeyImages - checks if wallet needs to import multisig key images from other participants + * @return true if there are partial key images + */ + virtual bool hasMultisigPartialKeyImages() const = 0; /** * @brief restoreMultisigTransaction creates PendingTransaction from signData @@ -857,10 +898,13 @@ struct Wallet virtual bool rescanSpent() = 0; //! blackballs a set of outputs - virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add) = 0; + virtual bool blackballOutputs(const std::vector<std::string> &outputs, bool add) = 0; + + //! blackballs an output + virtual bool blackballOutput(const std::string &amount, const std::string &offset) = 0; //! unblackballs an output - virtual bool unblackballOutput(const std::string &pubkey) = 0; + virtual bool unblackballOutput(const std::string &amount, const std::string &offset) = 0; //! gets the ring used for a key image, if any virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const = 0; @@ -885,6 +929,18 @@ struct Wallet //! Initiates a light wallet import wallet request virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status) = 0; + + //! locks/unlocks the keys file; returns true on success + virtual bool lockKeysFile() = 0; + virtual bool unlockKeysFile() = 0; + //! returns true if the keys file is locked + virtual bool isKeysFileLocked() = 0; + + /*! + * \brief Queries backing device for wallet keys + * \return Device they are on + */ + virtual Device getDeviceType() const = 0; }; /** @@ -899,9 +955,10 @@ struct WalletManager * \param password Password of wallet file * \param language Language to be used to generate electrum seed mnemonic * \param nettype Network type + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if created successfully) */ - virtual Wallet * createWallet(const std::string &path, const std::string &password, const std::string &language, NetworkType nettype) = 0; + virtual Wallet * createWallet(const std::string &path, const std::string &password, const std::string &language, NetworkType nettype, uint64_t kdf_rounds = 1) = 0; Wallet * createWallet(const std::string &path, const std::string &password, const std::string &language, bool testnet = false) // deprecated { return createWallet(path, password, language, testnet ? TESTNET : MAINNET); @@ -912,9 +969,10 @@ struct WalletManager * \param path Name of wallet file * \param password Password of wallet file * \param nettype Network type + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if opened successfully) */ - virtual Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype) = 0; + virtual Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype, uint64_t kdf_rounds = 1) = 0; Wallet * openWallet(const std::string &path, const std::string &password, bool testnet = false) // deprecated { return openWallet(path, password, testnet ? TESTNET : MAINNET); @@ -927,10 +985,11 @@ struct WalletManager * \param mnemonic mnemonic (25 words electrum seed) * \param nettype Network type * \param restoreHeight restore from start height + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) */ virtual Wallet * recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, - NetworkType nettype = MAINNET, uint64_t restoreHeight = 0) = 0; + NetworkType nettype = MAINNET, uint64_t restoreHeight = 0, uint64_t kdf_rounds = 1) = 0; Wallet * recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, bool testnet = false, uint64_t restoreHeight = 0) // deprecated { @@ -962,6 +1021,7 @@ struct WalletManager * \param addressString public address * \param viewKeyString view key * \param spendKeyString spend key (optional) + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) */ virtual Wallet * createWalletFromKeys(const std::string &path, @@ -971,7 +1031,8 @@ struct WalletManager uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString = "") = 0; + const std::string &spendKeyString = "", + uint64_t kdf_rounds = 1) = 0; Wallet * createWalletFromKeys(const std::string &path, const std::string &password, const std::string &language, @@ -1015,6 +1076,25 @@ struct WalletManager } /*! + * \brief creates wallet using hardware device. + * \param path Name of wallet file to be created + * \param password Password of wallet file + * \param nettype Network type + * \param deviceName Device name + * \param restoreHeight restore from start height (0 sets to current height) + * \param subaddressLookahead Size of subaddress lookahead (empty sets to some default low value) + * \param kdf_rounds Number of rounds for key derivation function + * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) + */ + virtual Wallet * createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight = 0, + const std::string &subaddressLookahead = "", + uint64_t kdf_rounds = 1) = 0; + + /*! * \brief Closes wallet. In case operation succeeded, wallet object deleted. in case operation failed, wallet object not deleted * \param wallet previously opened / created wallet instance * \return None @@ -1037,9 +1117,26 @@ struct WalletManager * @param keys_file_name - location of keys file * @param password - password to verify * @param no_spend_key - verify only view keys? + * @param kdf_rounds - number of rounds for key derivation function * @return - true if password is correct + * + * @note + * This function will fail when the wallet keys file is opened because the wallet program locks the keys file. + * In this case, Wallet::unlockKeysFile() and Wallet::lockKeysFile() need to be called before and after the call to this function, respectively. + */ + virtual bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key, uint64_t kdf_rounds = 1) const = 0; + + /*! + * \brief determine the key storage for the specified wallet file + * \param device_type (OUT) wallet backend as enumerated in Wallet::Device + * \param keys_file_name Keys file to verify password for + * \param password Password to verify + * \return true if password correct, else false + * + * for verification only - determines key storage hardware + * */ - virtual bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const = 0; + virtual bool queryWalletDevice(Wallet::Device& device_type, const std::string &keys_file_name, const std::string &password, uint64_t kdf_rounds = 1) const = 0; /*! * \brief findWallets - searches for the wallet files by given path name recursively diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index a63716576..5b262f1b7 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -50,16 +50,16 @@ namespace epee { namespace Monero { Wallet *WalletManagerImpl::createWallet(const std::string &path, const std::string &password, - const std::string &language, NetworkType nettype) + const std::string &language, NetworkType nettype, uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); wallet->create(path, password, language); return wallet; } -Wallet *WalletManagerImpl::openWallet(const std::string &path, const std::string &password, NetworkType nettype) +Wallet *WalletManagerImpl::openWallet(const std::string &path, const std::string &password, NetworkType nettype, uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); wallet->open(path, password); //Refresh addressBook wallet->addressBook()->refresh(); @@ -87,9 +87,10 @@ Wallet *WalletManagerImpl::recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, NetworkType nettype, - uint64_t restoreHeight) + uint64_t restoreHeight, + uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); } @@ -104,9 +105,10 @@ Wallet *WalletManagerImpl::createWalletFromKeys(const std::string &path, uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString) + const std::string &spendKeyString, + uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); } @@ -114,6 +116,27 @@ Wallet *WalletManagerImpl::createWalletFromKeys(const std::string &path, return wallet; } +Wallet *WalletManagerImpl::createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight, + const std::string &subaddressLookahead, + uint64_t kdf_rounds) +{ + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); + if(restoreHeight > 0){ + wallet->setRefreshFromBlockHeight(restoreHeight); + } + auto lookahead = tools::parse_subaddress_lookahead(subaddressLookahead); + if (lookahead) + { + wallet->setSubaddressLookahead(lookahead->first, lookahead->second); + } + wallet->recoverFromDevice(path, password, deviceName); + return wallet; +} + bool WalletManagerImpl::closeWallet(Wallet *wallet, bool store) { WalletImpl * wallet_ = dynamic_cast<WalletImpl*>(wallet); @@ -139,9 +162,17 @@ bool WalletManagerImpl::walletExists(const std::string &path) return false; } -bool WalletManagerImpl::verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const +bool WalletManagerImpl::verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key, uint64_t kdf_rounds) const +{ + return tools::wallet2::verify_password(keys_file_name, password, no_spend_key, hw::get_device("default"), kdf_rounds); +} + +bool WalletManagerImpl::queryWalletDevice(Wallet::Device& device_type, const std::string &keys_file_name, const std::string &password, uint64_t kdf_rounds) const { - return tools::wallet2::verify_password(keys_file_name, password, no_spend_key, hw::get_device("default")); + hw::device::device_type type; + bool r = tools::wallet2::query_device(type, keys_file_name, password, kdf_rounds); + device_type = static_cast<Wallet::Device>(type); + return r; } std::vector<std::string> WalletManagerImpl::findWallets(const std::string &path) diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index 26238b658..b3c0d6c00 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -39,13 +39,14 @@ class WalletManagerImpl : public WalletManager { public: Wallet * createWallet(const std::string &path, const std::string &password, - const std::string &language, NetworkType nettype); - Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype); + const std::string &language, NetworkType nettype, uint64_t kdf_rounds = 1) override; + Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype, uint64_t kdf_rounds = 1) override; virtual Wallet * recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, NetworkType nettype, - uint64_t restoreHeight); + uint64_t restoreHeight, + uint64_t kdf_rounds = 1) override; virtual Wallet * createWalletFromKeys(const std::string &path, const std::string &password, const std::string &language, @@ -53,9 +54,10 @@ public: uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString = ""); + const std::string &spendKeyString = "", + uint64_t kdf_rounds = 1) override; // next two methods are deprecated - use the above version which allow setting of a password - virtual Wallet * recoveryWallet(const std::string &path, const std::string &mnemonic, NetworkType nettype, uint64_t restoreHeight); + virtual Wallet * recoveryWallet(const std::string &path, const std::string &mnemonic, NetworkType nettype, uint64_t restoreHeight) override; // deprecated: use createWalletFromKeys(..., password, ...) instead virtual Wallet * createWalletFromKeys(const std::string &path, const std::string &language, @@ -63,23 +65,31 @@ public: uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString = ""); - virtual bool closeWallet(Wallet *wallet, bool store = true); - bool walletExists(const std::string &path); - bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const; - std::vector<std::string> findWallets(const std::string &path); - std::string errorString() const; - void setDaemonAddress(const std::string &address); - bool connected(uint32_t *version = NULL); - uint64_t blockchainHeight(); - uint64_t blockchainTargetHeight(); - uint64_t networkDifficulty(); - double miningHashRate(); - uint64_t blockTarget(); - bool isMining(); - bool startMining(const std::string &address, uint32_t threads = 1, bool background_mining = false, bool ignore_battery = true); - bool stopMining(); - std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const; + const std::string &spendKeyString = "") override; + virtual Wallet * createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight = 0, + const std::string &subaddressLookahead = "", + uint64_t kdf_rounds = 1) override; + virtual bool closeWallet(Wallet *wallet, bool store = true) override; + bool walletExists(const std::string &path) override; + bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key, uint64_t kdf_rounds = 1) const override; + bool queryWalletDevice(Wallet::Device& device_type, const std::string &keys_file_name, const std::string &password, uint64_t kdf_rounds = 1) const override; + std::vector<std::string> findWallets(const std::string &path) override; + std::string errorString() const override; + void setDaemonAddress(const std::string &address) override; + bool connected(uint32_t *version = NULL) override; + uint64_t blockchainHeight() override; + uint64_t blockchainTargetHeight() override; + uint64_t networkDifficulty() override; + double miningHashRate() override; + uint64_t blockTarget() override; + bool isMining() override; + bool startMining(const std::string &address, uint32_t threads = 1, bool background_mining = false, bool ignore_battery = true) override; + bool stopMining() override; + std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const override; private: WalletManagerImpl() {} diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index c5d869354..346c052b5 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -41,29 +41,23 @@ static const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::c NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, boost::mutex &mutex) : m_http_client(http_client) , m_daemon_rpc_mutex(mutex) - , m_height(0) - , m_height_time(0) - , m_earliest_height() - , m_dynamic_per_kb_fee_estimate(0) - , m_dynamic_per_kb_fee_estimate_cached_height(0) - , m_dynamic_per_kb_fee_estimate_grace_blocks(0) - , m_rpc_version(0) - , m_target_height(0) - , m_target_height_time(0) -{} +{ + invalidate(); +} void NodeRPCProxy::invalidate() { m_height = 0; - m_height_time = 0; for (size_t n = 0; n < 256; ++n) m_earliest_height[n] = 0; - m_dynamic_per_kb_fee_estimate = 0; - m_dynamic_per_kb_fee_estimate_cached_height = 0; - m_dynamic_per_kb_fee_estimate_grace_blocks = 0; + m_dynamic_base_fee_estimate = 0; + m_dynamic_base_fee_estimate_cached_height = 0; + m_dynamic_base_fee_estimate_grace_blocks = 0; + m_fee_quantization_mask = 1; m_rpc_version = 0; m_target_height = 0; - m_target_height_time = 0; + m_block_weight_limit = 0; + m_get_info_time = 0; } boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version) const @@ -75,7 +69,7 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_version", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); - CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(r, std::string("Failed to connect to daemon"), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get daemon RPC version"); m_rpc_version = resp_t.version; @@ -84,36 +78,15 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_height(uint64_t &height) const -{ - const time_t now = time(NULL); - if (m_height == 0 || now >= m_height_time + 30) // re-cache every 30 seconds - { - cryptonote::COMMAND_RPC_GET_HEIGHT::request req = AUTO_VAL_INIT(req); - cryptonote::COMMAND_RPC_GET_HEIGHT::response res = AUTO_VAL_INIT(res); - - m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json("/getheight", req, res, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(res.status != CORE_RPC_STATUS_BUSY, res.status, "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(res.status == CORE_RPC_STATUS_OK, res.status, "Failed to get current blockchain height"); - m_height = res.height; - m_height_time = now; - } - height = m_height; - return boost::optional<std::string>(); -} - void NodeRPCProxy::set_height(uint64_t h) { m_height = h; } -boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) const +boost::optional<std::string> NodeRPCProxy::get_info() const { const time_t now = time(NULL); - if (m_target_height == 0 || now >= m_target_height_time + 30) // re-cache every 30 seconds + if (now >= m_get_info_time + 30) // re-cache every 30 seconds { cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); @@ -122,16 +95,44 @@ boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) c bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); - CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(r, std::string("Failed to connect to daemon"), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get target blockchain height"); + m_height = resp_t.height; m_target_height = resp_t.target_height; - m_target_height_time = now; + m_block_weight_limit = resp_t.block_weight_limit ? resp_t.block_weight_limit : resp_t.block_size_limit; + m_get_info_time = now; } + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_height(uint64_t &height) const +{ + auto res = get_info(); + if (res) + return res; + height = m_height; + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) const +{ + auto res = get_info(); + if (res) + return res; height = m_target_height; return boost::optional<std::string>(); } +boost::optional<std::string> NodeRPCProxy::get_block_weight_limit(uint64_t &block_weight_limit) const +{ + auto res = get_info(); + if (res) + return res; + block_weight_limit = m_block_weight_limit; + return boost::optional<std::string>(); +} + boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version, uint64_t &earliest_height) const { if (m_earliest_height[version] == 0) @@ -143,17 +144,17 @@ boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version, req_t.version = version; bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); - CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(r, std::string("Failed to connect to daemon"), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get hard fork status"); - m_earliest_height[version] = resp_t.enabled ? resp_t.earliest_height : std::numeric_limits<uint64_t>::max(); + m_earliest_height[version] = resp_t.earliest_height; } earliest_height = m_earliest_height[version]; return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const +boost::optional<std::string> NodeRPCProxy::get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const { uint64_t height; @@ -161,24 +162,59 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 if (result) return result; - if (m_dynamic_per_kb_fee_estimate_cached_height != height || m_dynamic_per_kb_fee_estimate_grace_blocks != grace_blocks) + if (m_dynamic_base_fee_estimate_cached_height != height || m_dynamic_base_fee_estimate_grace_blocks != grace_blocks) { - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); req_t.grace_blocks = grace_blocks; bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_fee_estimate", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); - CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(r, std::string("Failed to connect to daemon"), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); - m_dynamic_per_kb_fee_estimate = resp_t.fee; - m_dynamic_per_kb_fee_estimate_cached_height = height; - m_dynamic_per_kb_fee_estimate_grace_blocks = grace_blocks; + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_dynamic_base_fee_estimate_grace_blocks = grace_blocks; + m_fee_quantization_mask = resp_t.quantization_mask; } - fee = m_dynamic_per_kb_fee_estimate; + fee = m_dynamic_base_fee_estimate; + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_fee_quantization_mask(uint64_t &fee_quantization_mask) const +{ + uint64_t height; + + boost::optional<std::string> result = get_height(height); + if (result) + return result; + + if (m_dynamic_base_fee_estimate_cached_height != height) + { + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + + m_daemon_rpc_mutex.lock(); + req_t.grace_blocks = m_dynamic_base_fee_estimate_grace_blocks; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_fee_estimate", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + CHECK_AND_ASSERT_MES(r, std::string("Failed to connect to daemon"), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_fee_quantization_mask = resp_t.quantization_mask; + } + + fee_quantization_mask = m_fee_quantization_mask; + if (fee_quantization_mask == 0) + { + MERROR("Fee quantization mask is 0, forcing to 1"); + fee_quantization_mask = 1; + } return boost::optional<std::string>(); } diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 1b183212d..65f13eaaa 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -47,22 +47,27 @@ public: boost::optional<std::string> get_height(uint64_t &height) const; void set_height(uint64_t h); boost::optional<std::string> get_target_height(uint64_t &height) const; + boost::optional<std::string> get_block_weight_limit(uint64_t &block_weight_limit) const; boost::optional<std::string> get_earliest_height(uint8_t version, uint64_t &earliest_height) const; - boost::optional<std::string> get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask) const; private: + boost::optional<std::string> get_info() const; + epee::net_utils::http::http_simple_client &m_http_client; boost::mutex &m_daemon_rpc_mutex; mutable uint64_t m_height; - mutable time_t m_height_time; mutable uint64_t m_earliest_height[256]; - mutable uint64_t m_dynamic_per_kb_fee_estimate; - mutable uint64_t m_dynamic_per_kb_fee_estimate_cached_height; - mutable uint64_t m_dynamic_per_kb_fee_estimate_grace_blocks; + mutable uint64_t m_dynamic_base_fee_estimate; + mutable uint64_t m_dynamic_base_fee_estimate_cached_height; + mutable uint64_t m_dynamic_base_fee_estimate_grace_blocks; + mutable uint64_t m_fee_quantization_mask; mutable uint32_t m_rpc_version; mutable uint64_t m_target_height; - mutable time_t m_target_height_time; + mutable uint64_t m_block_weight_limit; + mutable time_t m_get_info_time; }; } diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 3f2634c8b..f562d6c06 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -55,6 +55,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } +static int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + return va < vb ? -1 : va > vb; +} + static std::string compress_ring(const std::vector<uint64_t> &ring) { std::string s; @@ -146,7 +153,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed) MDB_stat mst; int ret; - needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB ret = mdb_env_info(env, &mei); if (ret) @@ -217,9 +224,9 @@ ringdb::ringdb(std::string filename, const std::string &genesis): THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_compare(txn, dbi_rings, compare_hash32); - dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); + dbr = mdb_dbi_open(txn, ("blackballs2-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - mdb_set_dupsort(txn, dbi_blackballs, compare_hash32); + mdb_set_dupsort(txn, dbi_blackballs, compare_uint64); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); @@ -374,7 +381,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::blackball_worker(const crypto::public_key &output, int op) +bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op) { MDB_txn *txn; MDB_cursor *cursor; @@ -382,49 +389,62 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) bool tx_active = false; bool ret = true; - dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output"); + + dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; - MDB_val key = zerokeyval; - MDB_val data; - data.mv_data = (void*)&output; - data.mv_size = sizeof(output); + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); - switch (op) + MDB_val key, data; + for (const std::pair<uint64_t, uint64_t> &output: outputs) { - case BLACKBALL_BLACKBALL: - MDEBUG("Blackballing output " << output); - dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); - if (dbr == MDB_KEYEXIST) - dbr = 0; - break; - case BLACKBALL_UNBLACKBALL: - MDEBUG("Unblackballing output " << output); - dbr = mdb_del(txn, dbi_blackballs, &key, &data); - if (dbr == MDB_NOTFOUND) - dbr = 0; - break; - case BLACKBALL_QUERY: - dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); - dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); - THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); - ret = dbr != MDB_NOTFOUND; - if (dbr == MDB_NOTFOUND) - dbr = 0; - mdb_cursor_close(cursor); - break; - case BLACKBALL_CLEAR: - dbr = mdb_drop(txn, dbi_blackballs, 0); - break; - default: - THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + key.mv_data = (void*)&output.first; + key.mv_size = sizeof(output.first); + data.mv_data = (void*)&output.second; + data.mv_size = sizeof(output.second); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Marking output " << output.first << "/" << output.second << " as spent"); + dbr = mdb_cursor_put(cursor, &key, &data, MDB_APPENDDUP); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Marking output " << output.first << "/" << output.second << " as unspent"); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + if (dbr == 0) + dbr = mdb_cursor_del(cursor, 0); + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_CLEAR: + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + } + + mdb_cursor_close(cursor); + + if (op == BLACKBALL_CLEAR) + { + dbr = mdb_drop(txn, dbi_blackballs, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr))); } - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); @@ -432,24 +452,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) return ret; } -bool ringdb::blackball(const crypto::public_key &output) +bool ringdb::blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs) +{ + return blackball_worker(outputs, BLACKBALL_BLACKBALL); +} + +bool ringdb::blackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_BLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_BLACKBALL); } -bool ringdb::unblackball(const crypto::public_key &output) +bool ringdb::unblackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_UNBLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_UNBLACKBALL); } -bool ringdb::blackballed(const crypto::public_key &output) +bool ringdb::blackballed(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_QUERY); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_QUERY); } bool ringdb::clear_blackballs() { - return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); + return blackball_worker(std::vector<std::pair<uint64_t, uint64_t>>(), BLACKBALL_CLEAR); } } diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 6b4bce124..7b448b0d7 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -49,13 +49,14 @@ namespace tools bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); - bool blackball(const crypto::public_key &output); - bool unblackball(const crypto::public_key &output); - bool blackballed(const crypto::public_key &output); + bool blackball(const std::pair<uint64_t, uint64_t> &output); + bool blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs); + bool unblackball(const std::pair<uint64_t, uint64_t> &output); + bool blackballed(const std::pair<uint64_t, uint64_t> &output); bool clear_blackballs(); private: - bool blackball_worker(const crypto::public_key &output, int op); + bool blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op); private: std::string filename; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index d2db45f12..c27e4e820 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -37,6 +37,8 @@ #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/split.hpp> +#include <boost/algorithm/string/join.hpp> +#include <boost/range/adaptor/transformed.hpp> #include "include_base_utils.h" using namespace epee; @@ -66,8 +68,11 @@ using namespace epee; #include "memwipe.h" #include "common/base58.h" #include "common/dns_utils.h" +#include "common/notify.h" #include "ringct/rctSigs.h" #include "ringdb.h" +#include "device/device_cold.hpp" +#include "device_trezor/device_trezor.hpp" extern "C" { @@ -84,11 +89,12 @@ using namespace cryptonote; // used to choose when to stop adding outputs to a tx #define APPROXIMATE_INPUT_BYTES 80 -// used to target a given block size (additional outputs may be added on top to build fee) -#define TX_SIZE_TARGET(bytes) (bytes*2/3) +// used to target a given block weight (additional outputs may be added on top to build fee) +#define TX_WEIGHT_TARGET(bytes) (bytes*2/3) // arbitrary, used to generate different hashes from the same input #define CHACHA8_KEY_TAIL 0x8c +#define CACHE_KEY_TAIL 0x8d #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\004" #define SIGNED_TX_PREFIX "Monero signed tx set\004" @@ -110,13 +116,19 @@ using namespace cryptonote; #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" -#define SEGREGATION_FORK_HEIGHT 1546000 -#define TESTNET_SEGREGATION_FORK_HEIGHT 1000000 -#define STAGENET_SEGREGATION_FORK_HEIGHT 1000000 +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" + +#define SEGREGATION_FORK_HEIGHT 99999999 +#define TESTNET_SEGREGATION_FORK_HEIGHT 99999999 +#define STAGENET_SEGREGATION_FORK_HEIGHT 99999999 #define SEGREGATION_FORK_VICINITY 1500 /* blocks */ -static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; +#define FIRST_REFRESH_GRANULARITY 1024 +#define GAMMA_PICK_HALF_WINDOW 5 + +static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; +static const std::string MULTISIG_EXTRA_INFO_MAGIC = "MultisigxV1"; namespace { @@ -128,6 +140,42 @@ namespace dir /= ".shared-ringdb"; return dir.string(); } + + std::string pack_multisignature_keys(const std::string& prefix, const std::vector<crypto::public_key>& keys, const crypto::secret_key& signer_secret_key) + { + std::string data; + crypto::public_key signer; + CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(signer_secret_key, signer), "Failed to derive public spend key"); + data += std::string((const char *)&signer, sizeof(crypto::public_key)); + + for (const auto &key: keys) + { + data += std::string((const char *)&key, sizeof(crypto::public_key)); + } + + data.resize(data.size() + sizeof(crypto::signature)); + + crypto::hash hash; + crypto::cn_fast_hash(data.data(), data.size() - sizeof(crypto::signature), hash); + crypto::signature &signature = *(crypto::signature*)&data[data.size() - sizeof(crypto::signature)]; + crypto::generate_signature(hash, signer, signer_secret_key, signature); + + return MULTISIG_EXTRA_INFO_MAGIC + tools::base58::encode(data); + } + + std::vector<crypto::public_key> secret_keys_to_public_keys(const std::vector<crypto::secret_key>& keys) + { + std::vector<crypto::public_key> public_keys; + public_keys.reserve(keys.size()); + + std::transform(keys.begin(), keys.end(), std::back_inserter(public_keys), [] (const crypto::secret_key& k) -> crypto::public_key { + crypto::public_key p; + CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(k, p), "Failed to derive public spend key"); + return p; + }); + + return public_keys; + } } namespace @@ -136,13 +184,14 @@ namespace struct options { const command_line::arg_descriptor<std::string> daemon_address = {"daemon-address", tools::wallet2::tr("Use daemon instance at <host>:<port>"), ""}; const command_line::arg_descriptor<std::string> daemon_host = {"daemon-host", tools::wallet2::tr("Use daemon instance at host <arg> instead of localhost"), ""}; + const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; + const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; const command_line::arg_descriptor<std::string> daemon_login = {"daemon-login", tools::wallet2::tr("Specify username[:password] for daemon RPC client"), "", true}; const command_line::arg_descriptor<bool> testnet = {"testnet", tools::wallet2::tr("For testnet. Daemon must also be launched with --testnet flag"), false}; const command_line::arg_descriptor<bool> stagenet = {"stagenet", tools::wallet2::tr("For stagenet. Daemon must also be launched with --stagenet flag"), false}; - const command_line::arg_descriptor<bool> restricted = {"restricted-rpc", tools::wallet2::tr("Restricts to view-only commands"), false}; const command_line::arg_descriptor<std::string, false, true, 2> shared_ringdb_dir = { "shared-ringdb-dir", tools::wallet2::tr("Set shared ring database path"), get_default_ringdb_path(), @@ -155,6 +204,9 @@ struct options { return val; } }; + const command_line::arg_descriptor<uint64_t> kdf_rounds = {"kdf-rounds", tools::wallet2::tr("Number of rounds for the key derivation function"), 1}; + const command_line::arg_descriptor<std::string> hw_device = {"hw-device", tools::wallet2::tr("HW device to use"), ""}; + const command_line::arg_descriptor<std::string> tx_notify = { "tx-notify" , "Run a program for each new incoming transaction, '%s' will be replaced by the transaction hash" , "" }; }; void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file) @@ -177,30 +229,35 @@ uint64_t calculate_fee(uint64_t fee_per_kb, size_t bytes, uint64_t fee_multiplie return kB * fee_per_kb * fee_multiplier; } -uint64_t calculate_fee(uint64_t fee_per_kb, const cryptonote::blobdata &blob, uint64_t fee_multiplier) +uint64_t calculate_fee_from_weight(uint64_t base_fee, uint64_t weight, uint64_t fee_multiplier, uint64_t fee_quantization_mask) { - return calculate_fee(fee_per_kb, blob.size(), fee_multiplier); + uint64_t fee = weight * base_fee * fee_multiplier; + fee = (fee + fee_quantization_mask - 1) / fee_quantization_mask * fee_quantization_mask; + return fee; } -std::string get_size_string(size_t sz) +std::string get_weight_string(size_t weight) { - return std::to_string(sz) + " bytes (" + std::to_string((sz + 1023) / 1024) + " kB)"; + return std::to_string(weight) + " weight"; } -std::string get_size_string(const cryptonote::blobdata &tx) +std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_size) { - return get_size_string(tx.size()); + return get_weight_string(get_transaction_weight(tx, blob_size)); } -std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); - const bool restricted = command_line::get_arg(vm, opts.restricted); + const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; + const uint64_t kdf_rounds = command_line::get_arg(vm, opts.kdf_rounds); + THROW_WALLET_EXCEPTION_IF(kdf_rounds == 0, tools::error::wallet_internal_error, "KDF rounds must not be 0"); auto daemon_address = command_line::get_arg(vm, opts.daemon_address); auto daemon_host = command_line::get_arg(vm, opts.daemon_host); auto daemon_port = command_line::get_arg(vm, opts.daemon_port); + auto device_name = command_line::get_arg(vm, opts.hw_device); THROW_WALLET_EXCEPTION_IF(!daemon_address.empty() && !daemon_host.empty() && 0 != daemon_port, tools::error::wallet_internal_error, tools::wallet2::tr("can't specify daemon host or port more than once")); @@ -224,16 +281,49 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (!daemon_port) { - daemon_port = testnet ? config::testnet::RPC_DEFAULT_PORT : stagenet ? config::stagenet::RPC_DEFAULT_PORT : config::RPC_DEFAULT_PORT; + daemon_port = get_config(nettype).RPC_DEFAULT_PORT; } if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); - std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(testnet ? TESTNET : stagenet ? STAGENET : MAINNET, restricted)); - wallet->init(std::move(daemon_address), std::move(login)); + boost::optional<bool> trusted_daemon; + if (!command_line::is_arg_defaulted(vm, opts.trusted_daemon) || !command_line::is_arg_defaulted(vm, opts.untrusted_daemon)) + trusted_daemon = command_line::get_arg(vm, opts.trusted_daemon) && !command_line::get_arg(vm, opts.untrusted_daemon); + THROW_WALLET_EXCEPTION_IF(!command_line::is_arg_defaulted(vm, opts.trusted_daemon) && !command_line::is_arg_defaulted(vm, opts.untrusted_daemon), + tools::error::wallet_internal_error, tools::wallet2::tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted")); + + // set --trusted-daemon if local and not overridden + if (!trusted_daemon) + { + try + { + trusted_daemon = false; + if (tools::is_local_address(daemon_address)) + { + MINFO(tools::wallet2::tr("Daemon is local, assuming trusted")); + trusted_daemon = true; + } + } + catch (const std::exception &e) { } + } + + std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds, unattended)); + wallet->init(std::move(daemon_address), std::move(login), 0, false, *trusted_daemon); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); + wallet->device_name(device_name); + + try + { + if (!command_line::is_arg_defaulted(vm, opts.tx_notify)) + wallet->set_tx_notify(std::shared_ptr<tools::Notify>(new tools::Notify(command_line::get_arg(vm, opts.tx_notify).c_str()))); + } + catch (const std::exception &e) + { + MERROR("Failed to parse tx notify spec"); + } + return wallet; } @@ -263,10 +353,10 @@ boost::optional<tools::password_container> get_password(const boost::program_opt THROW_WALLET_EXCEPTION_IF(!password_prompter, tools::error::wallet_internal_error, tools::wallet2::tr("no password specified; use --prompt-for-password to prompt for a password")); - return password_prompter(verify ? tr("Enter a new password for the wallet") : tr("Wallet password"), verify); + return password_prompter(verify ? tools::wallet2::tr("Enter a new password for the wallet") : tools::wallet2::tr("Wallet password"), verify); } -std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_from_json(const std::string& json_file, const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); @@ -276,6 +366,7 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, false. Gcc will coerce this into unique_ptr(nullptr), but clang correctly fails. This large wrapper is for the use of that macro */ std::unique_ptr<tools::wallet2> wallet; + epee::wipeable_string password; const auto do_generate = [&]() -> bool { std::string buf; if (!epee::file_io_utils::load_file_to_string(json_file, buf)) { @@ -404,7 +495,7 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, THROW_WALLET_EXCEPTION_IF(deprecated_wallet, tools::error::wallet_internal_error, tools::wallet2::tr("Cannot generate deprecated wallets from JSON")); - wallet.reset(make_basic(vm, opts, password_prompter).release()); + wallet.reset(make_basic(vm, unattended, opts, password_prompter).release()); wallet->set_refresh_from_block_height(field_scan_from_height); wallet->explicit_refresh_from_block_height(field_scan_from_height_found); @@ -413,10 +504,12 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, if (!field_seed.empty()) { wallet->generate(field_filename, field_password, recovery_key, recover, false, create_address_file); + password = field_password; } else if (field_viewkey.empty() && !field_spendkey.empty()) { wallet->generate(field_filename, field_password, spendkey, recover, false, create_address_file); + password = field_password; } else { @@ -443,6 +536,7 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, tools::wallet2::tr("Address must be specified in order to create watch-only wallet")); } wallet->generate(field_filename, field_password, address, viewkey, create_address_file); + password = field_password; } else { @@ -450,6 +544,7 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, tools::wallet2::tr("failed to verify spend key secret key")); } wallet->generate(field_filename, field_password, address, spendkey, viewkey, create_address_file); + password = field_password; } } } @@ -462,9 +557,9 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, if (do_generate()) { - return wallet; + return {std::move(wallet), tools::password_container(password)}; } - return nullptr; + return {nullptr, tools::password_container{}}; } static void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) @@ -547,7 +642,12 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // rangeSigs if (bulletproof) - size += ((2*6 + 4 + 5)*32 + 3) * n_outputs; + { + size_t log_padded_outputs = 0; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + size += (2 * (6 + log_padded_outputs) + 4 + 5) * 32 + 3; + } else size += (2*64*32+32+64*32) * n_outputs; @@ -566,23 +666,63 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // txnFee size += 4; - LOG_PRINT_L2("estimated rct tx size for " << n_inputs << " with ring size " << (mixin+1) << " and " << n_outputs << ": " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); + LOG_PRINT_L2("estimated " << (bulletproof ? "bulletproof" : "borromean") << " rct tx size for " << n_inputs << " inputs with ring size " << (mixin+1) << " and " << n_outputs << " outputs: " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); return size; } size_t estimate_tx_size(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) { if (use_rct) - return estimate_rct_tx_size(n_inputs, mixin, n_outputs + 1, extra_size, bulletproof); + return estimate_rct_tx_size(n_inputs, mixin, n_outputs, extra_size, bulletproof); else return n_inputs * (mixin+1) * APPROXIMATE_INPUT_BYTES + extra_size; } +uint64_t estimate_tx_weight(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) +{ + size_t size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + if (use_rct && bulletproof && n_outputs > 2) + { + const uint64_t bp_base = 368; + size_t log_padded_outputs = 2; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + uint64_t nlr = 2 * (6 + log_padded_outputs); + const uint64_t bp_size = 32 * (9 + nlr); + const uint64_t bp_clawback = (bp_base * (1<<log_padded_outputs) - bp_size) * 4 / 5; + MDEBUG("clawback on size " << size << ": " << bp_clawback); + size += bp_clawback; + } + return size; +} + uint8_t get_bulletproof_fork() { return 8; } +uint64_t estimate_fee(bool use_per_byte_fee, bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + { + const size_t estimated_tx_weight = estimate_tx_weight(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee_from_weight(base_fee, estimated_tx_weight, fee_multiplier, fee_quantization_mask); + } + else + { + const size_t estimated_tx_size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee(base_fee, estimated_tx_size, fee_multiplier); + } +} + +uint64_t calculate_fee(bool use_per_byte_fee, const cryptonote::transaction &tx, size_t blob_size, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + return calculate_fee_from_weight(base_fee, cryptonote::get_transaction_weight(tx, blob_size), fee_multiplier, fee_quantization_mask); + else + return calculate_fee(base_fee, blob_size, fee_multiplier); +} + crypto::hash8 get_short_payment_id(const tools::wallet2::pending_tx &ptx, hw::device &hwdev) { crypto::hash8 payment_id8 = null_hash8; @@ -630,6 +770,11 @@ uint32_t get_subaddress_clamped_sum(uint32_t idx, uint32_t extra) return idx + extra; } +static void setup_shim(hw::wallet_shim * shim, tools::wallet2 * wallet) +{ + shim->get_tx_pub_key_from_received_outs = boost::bind(&tools::wallet2::get_tx_pub_key_from_received_outs, wallet, _1); +} + //----------------------------------------------------------------- } //namespace @@ -641,12 +786,46 @@ const size_t MAX_SPLIT_ATTEMPTS = 30; constexpr const std::chrono::seconds wallet2::rpc_timeout; const char* wallet2::tr(const char* str) { return i18n_translate(str, "tools::wallet2"); } -wallet2::wallet2(network_type nettype, bool restricted): +wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, const boost::optional<tools::password_container> &password): + w(w), + locked(password != boost::none) +{ + if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt) + { + locked = false; + return; + } + const epee::wipeable_string pass = password->password(); + w.generate_chacha_key_from_password(pass, key); + w.decrypt_keys(key); +} + +wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, bool locked, const epee::wipeable_string &password): + w(w), + locked(locked) +{ + if (!locked) + return; + w.generate_chacha_key_from_password(password, key); + w.decrypt_keys(key); +} + +wallet_keys_unlocker::~wallet_keys_unlocker() +{ + if (!locked) + return; + w.encrypt_keys(key); +} + +wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), + m_upper_transaction_weight_limit(0), m_run(true), m_callback(0), + m_trusted_daemon(false), m_nettype(nettype), + m_multisig_rounds_passed(0), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), @@ -654,11 +833,12 @@ wallet2::wallet2(network_type nettype, bool restricted): m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), + m_first_refresh_done(false), m_refresh_from_block_height(0), m_explicit_refresh_from_block_height(true), m_confirm_missing_payment_id(true), m_confirm_non_default_ring_size(true), - m_ask_password(true), + m_ask_password(AskPasswordToDecrypt), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), @@ -669,9 +849,13 @@ wallet2::wallet2(network_type nettype, bool restricted): m_segregate_pre_fork_outputs(true), m_key_reuse_mitigation2(true), m_segregation_height(0), + m_ignore_fractional_outputs(true), m_is_initialized(false), - m_restricted(restricted), + m_kdf_rounds(kdf_rounds), is_old_file_format(false), + m_watch_only(false), + m_multisig(false), + m_multisig_threshold(0), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex), m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR), m_subaddress_lookahead_minor(SUBADDRESS_LOOKAHEAD_MINOR), @@ -681,9 +865,12 @@ wallet2::wallet2(network_type nettype, bool restricted): m_light_wallet_connected(false), m_light_wallet_balance(0), m_light_wallet_unlocked_balance(0), - m_key_on_device(false), + m_key_device_type(hw::device::device_type::SOFTWARE), m_ring_history_saved(false), - m_ringdb() + m_ringdb(), + m_last_block_reward(0), + m_encrypt_keys_after_refresh(boost::none), + m_unattended(unattended) { } @@ -701,29 +888,38 @@ bool wallet2::has_stagenet_option(const boost::program_options::variables_map& v return command_line::get_arg(vm, options().stagenet); } +std::string wallet2::device_name_option(const boost::program_options::variables_map& vm) +{ + return command_line::get_arg(vm, options().hw_device); +} + void wallet2::init_options(boost::program_options::options_description& desc_params) { const options opts{}; command_line::add_arg(desc_params, opts.daemon_address); command_line::add_arg(desc_params, opts.daemon_host); + command_line::add_arg(desc_params, opts.trusted_daemon); + command_line::add_arg(desc_params, opts.untrusted_daemon); command_line::add_arg(desc_params, opts.password); command_line::add_arg(desc_params, opts.password_file); command_line::add_arg(desc_params, opts.daemon_port); command_line::add_arg(desc_params, opts.daemon_login); command_line::add_arg(desc_params, opts.testnet); command_line::add_arg(desc_params, opts.stagenet); - command_line::add_arg(desc_params, opts.restricted); command_line::add_arg(desc_params, opts.shared_ringdb_dir); + command_line::add_arg(desc_params, opts.kdf_rounds); + command_line::add_arg(desc_params, opts.hw_device); + command_line::add_arg(desc_params, opts.tx_notify); } -std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::pair<std::unique_ptr<wallet2>, tools::password_container> wallet2::make_from_json(const boost::program_options::variables_map& vm, bool unattended, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; - return generate_from_json(json_file, vm, opts, password_prompter); + return generate_from_json(json_file, vm, unattended, opts, password_prompter); } std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( - const boost::program_options::variables_map& vm, const std::string& wallet_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) + const boost::program_options::variables_map& vm, bool unattended, const std::string& wallet_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; auto pwd = get_password(vm, opts, password_prompter, false); @@ -731,7 +927,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( { return {nullptr, password_container{}}; } - auto wallet = make_basic(vm, opts, password_prompter); + auto wallet = make_basic(vm, unattended, opts, password_prompter); if (wallet) { wallet->load(wallet_file, pwd->password()); @@ -739,7 +935,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( return {std::move(wallet), std::move(*pwd)}; } -std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter) +std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter) { const options opts{}; auto pwd = get_password(vm, opts, password_prompter, true); @@ -747,28 +943,27 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const { return {nullptr, password_container{}}; } - return {make_basic(vm, opts, password_prompter), std::move(*pwd)}; + return {make_basic(vm, unattended, opts, password_prompter), std::move(*pwd)}; } -std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::variables_map& vm, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; - return make_basic(vm, opts, password_prompter); + return make_basic(vm, unattended, opts, password_prompter); } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_size_limit, bool ssl) +bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool ssl, bool trusted_daemon) { m_checkpoints.init_default_checkpoints(m_nettype); if(m_http_client.is_connected()) m_http_client.disconnect(); m_is_initialized = true; - m_upper_transaction_size_limit = upper_transaction_size_limit; + m_upper_transaction_weight_limit = upper_transaction_weight_limit; m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); + m_trusted_daemon = trusted_daemon; // When switching from light wallet to full wallet, we need to reset the height we got from lw node. - if(m_light_wallet) - m_local_bc_height = m_blockchain.size(); return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl); } //---------------------------------------------------------------------------------------------------- @@ -777,11 +972,10 @@ bool wallet2::is_deterministic() const crypto::secret_key second; keccak((uint8_t *)&get_account().get_keys().m_spend_secret_key, sizeof(crypto::secret_key), (uint8_t *)&second, sizeof(crypto::secret_key)); sc_reduce32((uint8_t *)&second); - bool keys_deterministic = memcmp(second.data,get_account().get_keys().m_view_secret_key.data, sizeof(crypto::secret_key)) == 0; - return keys_deterministic; + return memcmp(second.data,get_account().get_keys().m_view_secret_key.data, sizeof(crypto::secret_key)) == 0; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_seed(std::string& electrum_words, const epee::wipeable_string &passphrase) const +bool wallet2::get_seed(epee::wipeable_string& electrum_words, const epee::wipeable_string &passphrase) const { bool keys_deterministic = is_deterministic(); if (!keys_deterministic) @@ -807,7 +1001,7 @@ bool wallet2::get_seed(std::string& electrum_words, const epee::wipeable_string return true; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string &passphrase, bool raw) const +bool wallet2::get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase, bool raw) const { bool ready; uint32_t threshold, total; @@ -830,7 +1024,7 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & crypto::secret_key skey; crypto::public_key pkey; const account_keys &keys = get_account().get_keys(); - std::string data; + epee::wipeable_string data; data.append((const char*)&threshold, sizeof(uint32_t)); data.append((const char*)&total, sizeof(uint32_t)); skey = keys.m_spend_secret_key; @@ -856,7 +1050,7 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & if (raw) { - seed = epee::string_tools::buff_to_hex_nodelimer(data); + seed = epee::to_hex::wipeable_string({(const unsigned char*)data.data(), data.size()}); } else { @@ -870,6 +1064,28 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & return true; } //---------------------------------------------------------------------------------------------------- +bool wallet2::reconnect_device() +{ + bool r = true; + hw::device &hwdev = lookup_device(m_device_name); + hwdev.set_name(m_device_name); + hwdev.set_network_type(m_nettype); + r = hwdev.init(); + if (!r){ + LOG_PRINT_L2("Could not init device"); + return false; + } + + r = hwdev.connect(); + if (!r){ + LOG_PRINT_L2("Could not connect to the device"); + return false; + } + + m_account.set_device(hwdev); + return true; +} +//---------------------------------------------------------------------------------------------------- /*! * \brief Gets the seed language */ @@ -892,6 +1108,14 @@ cryptonote::account_public_address wallet2::get_subaddress(const cryptonote::sub return hwdev.get_subaddress(m_account.get_keys(), index); } //---------------------------------------------------------------------------------------------------- +boost::optional<cryptonote::subaddress_index> wallet2::get_subaddress_index(const cryptonote::account_public_address& address) const +{ + auto index = m_subaddresses.find(address.m_spend_public_key); + if (index == m_subaddresses.end()) + return boost::none; + return index->second; +} +//---------------------------------------------------------------------------------------------------- crypto::public_key wallet2::get_subaddress_spend_public_key(const cryptonote::subaddress_index& index) const { hw::device &hwdev = m_account.get_device(); @@ -944,6 +1168,7 @@ void wallet2::expand_subaddresses(const cryptonote::subaddress_index& index) } m_subaddress_labels.resize(index.major + 1, {"Untitled account"}); m_subaddress_labels[index.major].resize(index.minor + 1); + get_account_tags(); } else if (m_subaddress_labels[index.major].size() <= index.minor) { @@ -1033,6 +1258,33 @@ void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivatio tx_scan_info.error = false; } //---------------------------------------------------------------------------------------------------- +void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const +{ + if (!is_out_data || i >= is_out_data->received.size()) + return check_acc_out_precomp(o, derivation, additional_derivations, i, tx_scan_info); + + tx_scan_info.received = is_out_data->received[i]; + if(tx_scan_info.received) + { + tx_scan_info.money_transfered = o.amount; // may be 0 for ringct outputs + } + else + { + tx_scan_info.money_transfered = 0; + } + tx_scan_info.error = false; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::check_acc_out_precomp_once(const tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const +{ + tx_scan_info.received = boost::none; + if (already_seen) + return; + check_acc_out_precomp(o, derivation, additional_derivations, i, is_out_data, tx_scan_info); + if (tx_scan_info.received) + already_seen = true; +} +//---------------------------------------------------------------------------------------------------- static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation &derivation, unsigned int i, rct::key & mask, hw::device &hwdev) { crypto::secret_key scalar1; @@ -1042,10 +1294,9 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & switch (rv.type) { case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: return rct::decodeRctSimple(rv, rct::sk2rct(scalar1), i, mask, hwdev); case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: return rct::decodeRct(rv, rct::sk2rct(scalar1), i, mask, hwdev); default: LOG_ERROR("Unsupported rct type: " << rv.type); @@ -1059,9 +1310,25 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & } } //---------------------------------------------------------------------------------------------------- -void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) const +void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) { THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); + + // if keys are encrypted, ask for password + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_multisig_rescan_k) + { + static critical_section password_lock; + CRITICAL_REGION_LOCAL(password_lock); + if (!m_encrypt_keys_after_refresh) + { + boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password("output received"); + THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); + THROW_WALLET_EXCEPTION_IF(!verify_password(*pwd), error::password_needed, tr("Invalid password: password is needed to compute key image for incoming monero")); + decrypt_keys(*pwd); + m_encrypt_keys_after_refresh = *pwd; + } + } + if (m_multisig) { tx_scan_info.in_ephemeral.pub = boost::get<cryptonote::txout_to_key>(tx.vout[i].target).key; @@ -1086,35 +1353,74 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi ++num_vouts_received; } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen) +void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const { - //ensure device is let in NONE mode in any case - hw::device &hwdev = m_account.get_device(); - - boost::unique_lock<hw::device> hwdev_lock (hwdev); - hw::reset_mode rst(hwdev); - hwdev_lock.unlock(); + const cryptonote::account_keys& keys = m_account.get_keys(); + + if(!parse_tx_extra(tx.extra, tx_cache_data.tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + tx_cache_data.tx_extra_fields.clear(); + return; + } - // In this function, tx (probably) only contains the base information + // Don't try to extract tx public key if tx has no ouputs + const bool is_miner = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!is_miner || m_refresh_type != RefreshType::RefreshNoCoinbase) + { + const size_t rec_size = is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : tx.vout.size(); + if (!tx.vout.empty()) + { + // if tx.vout is not empty, we loop through all tx pubkeys + const std::vector<boost::optional<cryptonote::subaddress_receive_info>> rec(rec_size, boost::none); + + tx_extra_pub_key pub_key_field; + size_t pk_index = 0; + while (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, pub_key_field, pk_index++)) + tx_cache_data.primary.push_back({pub_key_field.pub_key, {}, rec}); + + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + tx_extra_additional_pub_keys additional_tx_pub_keys; + std::vector<crypto::key_derivation> additional_derivations; + if (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + tx_cache_data.additional.push_back({additional_tx_pub_keys.data[i], {}, {}}); + } + } + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data) +{ + // In this function, tx (probably) only contains the base information // (that is, the prunable stuff may or may not be included) if (!miner_tx && !pool) process_unconfirmed(txid, tx, height); - std::vector<size_t> outs; std::unordered_map<cryptonote::subaddress_index, uint64_t> tx_money_got_in_outs; // per receiving subaddress index crypto::public_key tx_pub_key = null_pkey; + bool notify = false; - std::vector<tx_extra_field> tx_extra_fields; - if(!parse_tx_extra(tx.extra, tx_extra_fields)) + std::vector<tx_extra_field> local_tx_extra_fields; + if (tx_cache_data.tx_extra_fields.empty()) { - // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key - LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + if(!parse_tx_extra(tx.extra, local_tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + } } + const std::vector<tx_extra_field> &tx_extra_fields = tx_cache_data.tx_extra_fields.empty() ? local_tx_extra_fields : tx_cache_data.tx_extra_fields; // Don't try to extract tx public key if tx has no ouputs size_t pk_index = 0; std::vector<tx_scan_info_t> tx_scan_info(tx.vout.size()); + std::deque<bool> output_found(tx.vout.size(), false); + uint64_t total_received_1 = 0; while (!tx.vout.empty()) { + std::vector<size_t> outs; // if tx.vout is not empty, we loop through all tx pubkeys tx_extra_pub_key pub_key_field; @@ -1127,6 +1433,11 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote m_callback->on_skip_transaction(height, txid, tx); break; } + if (!tx_cache_data.primary.empty()) + { + THROW_WALLET_EXCEPTION_IF(tx_cache_data.primary.size() < pk_index || pub_key_field.pub_key != tx_cache_data.primary[pk_index - 1].pkey, + error::wallet_internal_error, "tx_cache_data is out of sync"); + } int num_vouts_received = 0; tx_pub_key = pub_key_field.pub_key; @@ -1135,28 +1446,55 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote const cryptonote::account_keys& keys = m_account.get_keys(); crypto::key_derivation derivation; - hwdev_lock.lock(); - hwdev.set_mode(hw::device::TRANSACTION_PARSE); - if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + std::vector<crypto::key_derivation> additional_derivations; + tx_extra_additional_pub_keys additional_tx_pub_keys; + const wallet2::is_out_data *is_out_data_ptr = NULL; + if (tx_cache_data.primary.empty()) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); - memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); - } + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); + hw::reset_mode rst(hwdev); - // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses - std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); - std::vector<crypto::key_derivation> additional_derivations; - for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey in " << txid << ", skipping"); + static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); + } + + if (pk_index == 1) + { + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + if (find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + { + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys.data[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from additional tx pubkey in " << txid << ", skipping"); + memcpy(&additional_derivations.back(), rct::identity().bytes, sizeof(crypto::key_derivation)); + } + } + } + } + } + else { - additional_derivations.push_back({}); - if (!hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back())) + THROW_WALLET_EXCEPTION_IF(pk_index - 1 >= tx_cache_data.primary.size(), + error::wallet_internal_error, "pk_index out of range of tx_cache_data"); + is_out_data_ptr = &tx_cache_data.primary[pk_index - 1]; + derivation = tx_cache_data.primary[pk_index - 1].derivation; + if (pk_index == 1) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - additional_derivations.pop_back(); + for (size_t n = 0; n < tx_cache_data.additional.size(); ++n) + { + additional_tx_pub_keys.data.push_back(tx_cache_data.additional[n].pkey); + additional_derivations.push_back(tx_cache_data.additional[n].derivation); + } } } - hwdev_lock.unlock(); if (miner_tx && m_refresh_type == RefreshNoCoinbase) { @@ -1164,7 +1502,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase) { - check_acc_out_precomp(tx.vout[0], derivation, additional_derivations, 0, tx_scan_info[0]); + check_acc_out_precomp_once(tx.vout[0], derivation, additional_derivations, 0, is_out_data_ptr, tx_scan_info[0], output_found[0]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[0].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); // this assumes that the miner tx pays a single address @@ -1174,60 +1512,60 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote // the first one was already checked for (size_t i = 1; i < tx.vout.size(); ++i) { - tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp_once, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i]), std::ref(output_found[i])), true); } - waiter.wait(); + waiter.wait(&tpool); // then scan all outputs from 0 - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } } - else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1) + else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1 && !is_out_data_ptr) { for (size_t i = 0; i < tx.vout.size(); ++i) { - tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp_once, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i]), std::ref(output_found[i])), true); } - waiter.wait(); + waiter.wait(&tpool); - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } else { for (size_t i = 0; i < tx.vout.size(); ++i) { - check_acc_out_precomp(tx.vout[i], derivation, additional_derivations, i, tx_scan_info[i]); + check_acc_out_precomp_once(tx.vout[i], derivation, additional_derivations, i, is_out_data_ptr, tx_scan_info[i], output_found[i]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); - hwdev_lock.unlock(); } } } @@ -1255,6 +1593,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote + ", m_transfers.size() is " + boost::lexical_cast<std::string>(m_transfers.size())); if (kit == m_pub_keys.end()) { + uint64_t amount = tx.vout[o].amount ? tx.vout[o].amount : tx_scan_info[o].amount; if (!pool) { m_transfers.push_back(boost::value_initialized<transfer_details>()); @@ -1267,14 +1606,13 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; td.m_key_image_partial = m_multisig; - td.m_amount = tx.vout[o].amount; + td.m_amount = amount; td.m_pk_index = pk_index - 1; td.m_subaddr_index = tx_scan_info[o].received->index; expand_subaddresses(tx_scan_info[o].received->index); - if (td.m_amount == 0) + if (tx.vout[o].amount == 0) { td.m_mask = tx_scan_info[o].mask; - td.m_amount = tx_scan_info[o].amount; td.m_rct = true; } else if (miner_tx && tx.version == 2) @@ -1302,22 +1640,33 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (0 != m_callback) m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index); } + total_received_1 += amount; + notify = true; } - else if (m_transfers[kit->second].m_spent || m_transfers[kit->second].amount() >= tx.vout[o].amount) + else if (m_transfers[kit->second].m_spent || m_transfers[kit->second].amount() >= tx_scan_info[o].amount) { LOG_ERROR("Public key " << epee::string_tools::pod_to_hex(kit->first) - << " from received " << print_money(tx.vout[o].amount) << " output already exists with " + << " from received " << print_money(tx_scan_info[o].amount) << " output already exists with " << (m_transfers[kit->second].m_spent ? "spent" : "unspent") << " " - << print_money(m_transfers[kit->second].amount()) << ", received output ignored"); + << print_money(m_transfers[kit->second].amount()) << " in tx " << m_transfers[kit->second].m_txid << ", received output ignored"); + THROW_WALLET_EXCEPTION_IF(tx_money_got_in_outs[tx_scan_info[o].received->index] < tx_scan_info[o].amount, + error::wallet_internal_error, "Unexpected values of new and old outputs"); + tx_money_got_in_outs[tx_scan_info[o].received->index] -= tx_scan_info[o].amount; } else { LOG_ERROR("Public key " << epee::string_tools::pod_to_hex(kit->first) - << " from received " << print_money(tx.vout[o].amount) << " output already exists with " + << " from received " << print_money(tx_scan_info[o].amount) << " output already exists with " << print_money(m_transfers[kit->second].amount()) << ", replacing with new output"); // The new larger output replaced a previous smaller one - tx_money_got_in_outs[tx_scan_info[o].received->index] -= tx.vout[o].amount; - + THROW_WALLET_EXCEPTION_IF(tx_money_got_in_outs[tx_scan_info[o].received->index] < tx_scan_info[o].amount, + error::wallet_internal_error, "Unexpected values of new and old outputs"); + THROW_WALLET_EXCEPTION_IF(m_transfers[kit->second].amount() > tx_scan_info[o].amount, + error::wallet_internal_error, "Unexpected values of new and old outputs"); + tx_money_got_in_outs[tx_scan_info[o].received->index] -= m_transfers[kit->second].amount(); + + uint64_t amount = tx.vout[o].amount ? tx.vout[o].amount : tx_scan_info[o].amount; + uint64_t extra_amount = amount - m_transfers[kit->second].amount(); if (!pool) { transfer_details &td = m_transfers[kit->second]; @@ -1326,14 +1675,13 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_global_output_index = o_indices[o]; td.m_tx = (const cryptonote::transaction_prefix&)tx; td.m_txid = txid; - td.m_amount = tx.vout[o].amount; + td.m_amount = amount; td.m_pk_index = pk_index - 1; td.m_subaddr_index = tx_scan_info[o].received->index; expand_subaddresses(tx_scan_info[o].received->index); - if (td.m_amount == 0) + if (tx.vout[o].amount == 0) { td.m_mask = tx_scan_info[o].mask; - td.m_amount = tx_scan_info[o].amount; td.m_rct = true; } else if (miner_tx && tx.version == 2) @@ -1360,6 +1708,8 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (0 != m_callback) m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index); } + total_received_1 += extra_amount; + notify = true; } } } @@ -1433,10 +1783,14 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } // remove change sent to the spending subaddress account from the list of received funds + uint64_t sub_change = 0; for (auto i = tx_money_got_in_outs.begin(); i != tx_money_got_in_outs.end();) { if (subaddr_account && i->first.major == *subaddr_account) + { + sub_change += i->second; i = tx_money_got_in_outs.erase(i); + } else ++i; } @@ -1453,6 +1807,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { // We got a payment ID to go with this tx LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8); + MINFO("Consider using subaddresses instead of encrypted payment IDs"); if (tx_pub_key != null_pkey) { if (!m_account.get_device().decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key)) @@ -1476,11 +1831,22 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); + MWARNING("Found unencrypted payment ID: these are bad for privacy, consider using subaddresses instead"); } } - else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) - { - LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); + + uint64_t total_received_2 = sub_change; + for (const auto& i : tx_money_got_in_outs) + total_received_2 += i.second; + if (total_received_1 != total_received_2) + { + const el::Level level = el::Level::Warning; + MCLOG_RED(level, "global", "**********************************************************************"); + MCLOG_RED(level, "global", "Consistency failure in amounts received"); + MCLOG_RED(level, "global", "Check transaction " << txid); + MCLOG_RED(level, "global", "**********************************************************************"); + exit(1); + return; } for (const auto& i : tx_money_got_in_outs) @@ -1492,6 +1858,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote payment.m_block_height = height; payment.m_unlock_time = tx.unlock_time; payment.m_timestamp = ts; + payment.m_coinbase = miner_tx; payment.m_subaddr_index = i.first; if (pool) { emplace_or_replace(m_unconfirmed_payments, payment_id, pool_payment_details{payment, double_spend_seen}); @@ -1503,6 +1870,13 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote LOG_PRINT_L2("Payment found in " << (pool ? "pool" : "block") << ": " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount); } } + + if (notify) + { + std::shared_ptr<tools::Notify> tx_notify = m_tx_notify; + if (tx_notify) + tx_notify->notify(epee::string_tools::pod_to_hex(txid).c_str()); + } } //---------------------------------------------------------------------------------------------------- void wallet2::process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height) @@ -1567,12 +1941,11 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans add_rings(tx); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices) +void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset) { - size_t txidx = 0; - THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != o_indices.indices.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != parsed_block.o_indices.indices.size(), error::wallet_internal_error, "block transactions=" + std::to_string(bche.txs.size()) + - " not match with daemon response size=" + std::to_string(o_indices.indices.size())); + " not match with daemon response size=" + std::to_string(parsed_block.o_indices.indices.size())); //handle transactions from new block @@ -1580,39 +1953,38 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height) { TIME_MEASURE_START(miner_tx_handle_time); - process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, o_indices.indices[txidx++].indices, height, b.timestamp, true, false, false); + if (m_refresh_type != RefreshNoCoinbase) + process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset]); + ++tx_cache_data_offset; TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); THROW_WALLET_EXCEPTION_IF(bche.txs.size() != b.tx_hashes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); - size_t idx = 0; - for (const auto& txblob: bche.txs) + THROW_WALLET_EXCEPTION_IF(bche.txs.size() != parsed_block.txes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); + for (size_t idx = 0; idx < b.tx_hashes.size(); ++idx) { - cryptonote::transaction tx; - bool r = parse_and_validate_tx_base_from_blob(txblob, tx); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob); - process_new_transaction(b.tx_hashes[idx], tx, o_indices.indices[txidx++].indices, height, b.timestamp, false, false, false); - ++idx; + process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++]); } TIME_MEASURE_FINISH(txs_handle_time); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); }else { - if (!(height % 100)) + if (!(height % 128)) LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); } m_blockchain.push_back(bl_id); - ++m_local_bc_height; if (0 != m_callback) m_callback->on_new_block(height, b); } //---------------------------------------------------------------------------------------------------- -void wallet2::get_short_chain_history(std::list<crypto::hash>& ids) const +void wallet2::get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity) const { size_t i = 0; size_t current_multiplier = 1; - size_t sz = m_blockchain.size() - m_blockchain.offset(); + size_t blockchain_size = std::max((size_t)(m_blockchain.size() / granularity * granularity), m_blockchain.offset()); + size_t sz = blockchain_size - m_blockchain.offset(); if(!sz) { ids.push_back(m_blockchain.genesis()); @@ -1647,41 +2019,15 @@ void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::bl bl_id = get_block_hash(bl); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices) +void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices) { cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); req.block_ids = short_chain_history; - uint32_t rpc_version; - boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); - // no error - if (!!result) - { - // empty string -> not connection - THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion"); - THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion"); - if (*result != CORE_RPC_STATUS_OK) - { - MDEBUG("Cannot determine daemon RPC version, not asking for pruned blocks"); - req.prune = false; // old daemon - } - } - else - { - if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 7)) - { - MDEBUG("Daemon is recent enough, asking for pruned blocks"); - req.prune = true; - } - else - { - MDEBUG("Daemon is too old, not asking for pruned blocks"); - req.prune = false; - } - } - + req.prune = true; req.start_height = start_height; + req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -1693,11 +2039,11 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, boost::lexical_cast<std::string>(res.output_indices.size()) + ") sizes from daemon"); blocks_start_height = res.start_height; - blocks = res.blocks; - o_indices = res.output_indices; + blocks = std::move(res.blocks); + o_indices = std::move(res.output_indices); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes) +void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes) { cryptonote::COMMAND_RPC_GET_HASHES_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_HASHES_FAST::response res = AUTO_VAL_INIT(res); @@ -1712,88 +2058,117 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); blocks_start_height = res.start_height; - hashes = res.m_block_ids; + hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, const std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t& blocks_added) +void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added) { size_t current_index = start_height; blocks_added = 0; - size_t tx_o_indices_idx = 0; - THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "size mismatch"); - THROW_WALLET_EXCEPTION_IF(!m_blockchain.is_in_bounds(current_index), error::wallet_internal_error, "Index out of bounds of hashchain"); + THROW_WALLET_EXCEPTION_IF(blocks.size() != parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); + THROW_WALLET_EXCEPTION_IF(!m_blockchain.is_in_bounds(current_index), error::out_of_hashchain_bounds_error); tools::threadpool& tpool = tools::threadpool::getInstance(); - int threads = tpool.get_max_concurrency(); - if (threads > 1) + tools::threadpool::waiter waiter; + + size_t num_txes = 0; + std::vector<tx_cache_data> tx_cache_data; + for (size_t i = 0; i < blocks.size(); ++i) + num_txes += 1 + parsed_blocks[i].txes.size(); + tx_cache_data.resize(num_txes); + size_t txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - std::vector<crypto::hash> round_block_hashes(threads); - std::vector<cryptonote::block> round_blocks(threads); - std::deque<bool> error(threads); - size_t blocks_size = blocks.size(); - std::list<block_complete_entry>::const_iterator blocki = blocks.begin(); - for (size_t b = 0; b < blocks_size; b += threads) + THROW_WALLET_EXCEPTION_IF(parsed_blocks[i].txes.size() != parsed_blocks[i].block.tx_hashes.size(), + error::wallet_internal_error, "Mismatched parsed_blocks[i].txes.size() and parsed_blocks[i].block.tx_hashes.size()"); + if (m_refresh_type != RefreshNoCoinbase) + tpool.submit(&waiter, [&, i, txidx](){ cache_tx_data(parsed_blocks[i].block.miner_tx, get_transaction_hash(parsed_blocks[i].block.miner_tx), tx_cache_data[txidx]); }); + ++txidx; + for (size_t idx = 0; idx < parsed_blocks[i].txes.size(); ++idx) { - size_t round_size = std::min((size_t)threads, blocks_size - b); - tools::threadpool::waiter waiter; + tpool.submit(&waiter, [&, i, idx, txidx](){ cache_tx_data(parsed_blocks[i].txes[idx], parsed_blocks[i].block.tx_hashes[idx], tx_cache_data[txidx]); }); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != num_txes, error::wallet_internal_error, "txidx does not match tx_cache_data size"); + waiter.wait(&tpool); - std::list<block_complete_entry>::const_iterator tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(tmpblocki->block), - std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i]))); - ++tmpblocki; - } - waiter.wait(); - tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, tmpblocki->block); - ++tmpblocki; - } - for (size_t i = 0; i < round_size; ++i) - { - const crypto::hash &bl_id = round_block_hashes[i]; - cryptonote::block &bl = round_blocks[i]; + hw::device &hwdev = m_account.get_device(); + hw::reset_mode rst(hwdev); + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + const cryptonote::account_keys &keys = m_account.get_keys(); - if(current_index >= m_blockchain.size()) - { - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - ++blocks_added; - } - else if(bl_id != m_blockchain[current_index]) - { - //split detected here !!! - THROW_WALLET_EXCEPTION_IF(current_index == start_height, error::wallet_internal_error, - "wrong daemon response: split starts from the first block in response " + string_tools::pod_to_hex(bl_id) + - " (height " + std::to_string(start_height) + "), local block id at this height: " + - string_tools::pod_to_hex(m_blockchain[current_index])); - - detach_blockchain(current_index); - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - } - else + auto gender = [&](wallet2::is_out_data &iod) { + boost::unique_lock<hw::device> hwdev_lock(hwdev); + if (!hwdev.generate_key_derivation(iod.pkey, keys.m_view_secret_key, iod.derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey, skipping"); + static_assert(sizeof(iod.derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&iod.derivation, rct::identity().bytes, sizeof(iod.derivation)); + } + }; + + for (auto &slot: tx_cache_data) + { + for (auto &iod: slot.primary) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + for (auto &iod: slot.additional) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + } + waiter.wait(&tpool); + + auto geniod = [&](const cryptonote::transaction &tx, size_t n_vouts, size_t txidx) { + for (size_t k = 0; k < n_vouts; ++k) + { + const auto &o = tx.vout[k]; + if (o.target.type() == typeid(cryptonote::txout_to_key)) + { + std::vector<crypto::key_derivation> additional_derivations; + for (const auto &iod: tx_cache_data[txidx].additional) + additional_derivations.push_back(iod.derivation); + const auto &key = boost::get<txout_to_key>(o.target).key; + for (size_t l = 0; l < tx_cache_data[txidx].primary.size(); ++l) { - LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); + THROW_WALLET_EXCEPTION_IF(tx_cache_data[txidx].primary[l].received.size() != n_vouts, + error::wallet_internal_error, "Unexpected received array size"); + tx_cache_data[txidx].primary[l].received[k] = is_out_to_acc_precomp(m_subaddresses, key, tx_cache_data[txidx].primary[l].derivation, additional_derivations, k, hwdev); + additional_derivations.clear(); } - ++current_index; - ++blocki; } } - } - else + }; + + txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - for(auto& bl_entry: blocks) + if (m_refresh_type != RefreshType::RefreshNoCoinbase) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); + tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); + } + ++txidx; + for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + tpool.submit(&waiter, [&, i, j, txidx](){ geniod(parsed_blocks[i].txes[j], parsed_blocks[i].txes[j].vout.size(), txidx); }, true); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != tx_cache_data.size(), error::wallet_internal_error, "txidx did not reach expected value"); + waiter.wait(&tpool); + hwdev.set_mode(hw::device::NONE); + + size_t tx_cache_data_offset = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - cryptonote::block bl; - bool r = cryptonote::parse_and_validate_block_from_blob(bl_entry.block, bl); - THROW_WALLET_EXCEPTION_IF(!r, error::block_parse_error, bl_entry.block); + const crypto::hash &bl_id = parsed_blocks[i].hash; + const cryptonote::block &bl = parsed_blocks[i].block; - crypto::hash bl_id = get_block_hash(bl); if(current_index >= m_blockchain.size()) { - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index, o_indices[tx_o_indices_idx]); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -1805,32 +2180,30 @@ void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote:: string_tools::pod_to_hex(m_blockchain[current_index])); detach_blockchain(current_index); - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index, o_indices[tx_o_indices_idx]); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); } else { LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); } - ++current_index; - ++tx_o_indices_idx; - } + tx_cache_data_offset += 1 + parsed_blocks[i].txes.size(); } } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh() +void wallet2::refresh(bool trusted_daemon) { uint64_t blocks_fetched = 0; - refresh(0, blocks_fetched); + refresh(trusted_daemon, 0, blocks_fetched); } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched) +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched) { bool received_money = false; - refresh(start_height, blocks_fetched, received_money); + refresh(trusted_daemon, start_height, blocks_fetched, received_money); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, bool &error) +void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error) { error = false; @@ -1838,19 +2211,56 @@ void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_hei { drop_from_short_history(short_chain_history, 3); + THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); + // prepend the last 3 blocks, should be enough to guard against a block or two's reorg - cryptonote::block bl; - std::list<cryptonote::block_complete_entry>::const_reverse_iterator i = prev_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) + std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); + for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) { - bool ok = cryptonote::parse_and_validate_block_from_blob(i->block, bl); - THROW_WALLET_EXCEPTION_IF(!ok, error::block_parse_error, i->block); - short_chain_history.push_front(cryptonote::get_block_hash(bl)); + short_chain_history.push_front(i->hash); ++i; } // pull the new blocks + std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices; pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); + THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "Mismatched sizes of blocks and o_indices"); + + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + parsed_blocks.resize(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) + { + tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(blocks[i].block), + std::ref(parsed_blocks[i].block), std::ref(parsed_blocks[i].hash), std::ref(parsed_blocks[i].error)), true); + } + waiter.wait(&tpool); + for (size_t i = 0; i < blocks.size(); ++i) + { + if (parsed_blocks[i].error) + { + error = true; + break; + } + parsed_blocks[i].o_indices = std::move(o_indices[i]); + } + + boost::mutex error_lock; + for (size_t i = 0; i < blocks.size(); ++i) + { + parsed_blocks[i].txes.resize(blocks[i].txs.size()); + for (size_t j = 0; j < blocks[i].txs.size(); ++j) + { + tpool.submit(&waiter, [&, i, j](){ + if (!parse_and_validate_tx_base_from_blob(blocks[i].txs[j], parsed_blocks[i].txes[j])) + { + boost::unique_lock<boost::mutex> lock(error_lock); + error = true; + } + }, true); + } + } + waiter.wait(&tpool); } catch(...) { @@ -1890,9 +2300,17 @@ void wallet2::update_pool_state(bool refreshed) { MDEBUG("update_pool_state start"); + auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { + if (m_encrypt_keys_after_refresh) + { + encrypt_keys(*m_encrypt_keys_after_refresh); + m_encrypt_keys_after_refresh = boost::none; + } + }); + // get the pool state - cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request req; - cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response res; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response res; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2063,7 +2481,7 @@ void wallet2::update_pool_state(bool refreshed) [tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; }); if (i != txids.end()) { - process_new_transaction(tx_hash, tx, std::vector<uint64_t>(), 0, time(NULL), false, true, tx_entry.double_spend_seen); + process_new_transaction(tx_hash, tx, std::vector<uint64_t>(), 0, time(NULL), false, true, tx_entry.double_spend_seen, {}); m_scanned_pool_txs[0].insert(tx_hash); if (m_scanned_pool_txs[0].size() > 5000) { @@ -2105,19 +2523,19 @@ void wallet2::update_pool_state(bool refreshed) MDEBUG("update_pool_state end"); } //---------------------------------------------------------------------------------------------------- -void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history) +void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force) { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; const uint64_t checkpoint_height = m_checkpoints.get_max_height(); - if (stop_height > checkpoint_height && m_blockchain.size()-1 < checkpoint_height) + if ((stop_height > checkpoint_height && m_blockchain.size()-1 < checkpoint_height) && !force) { // we will drop all these, so don't bother getting them uint64_t missing_blocks = m_checkpoints.get_max_height() - m_blockchain.size(); while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); - m_local_bc_height = m_blockchain.size(); + m_blockchain.trim(checkpoint_height); short_chain_history.clear(); get_short_chain_history(short_chain_history); } @@ -2135,7 +2553,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, } if (hashes.size() + current_index < stop_height) { drop_from_short_history(short_chain_history, 3); - std::list<crypto::hash>::iterator right = hashes.end(); + std::vector<crypto::hash>::iterator right = hashes.end(); // prepend 3 more for (int i = 0; i<3; i++) { right--; @@ -2147,10 +2565,9 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, { if(current_index >= m_blockchain.size()) { - if (!(current_index % 1000)) + if (!(current_index % 1024)) LOG_PRINT_L2( "Skipped block by height: " << current_index); m_blockchain.push_back(bl_id); - ++m_local_bc_height; if (0 != m_callback) { // FIXME: this isn't right, but simplewallet just logs that we got a block. @@ -2196,7 +2613,7 @@ bool wallet2::delete_address_book_row(std::size_t row_id) { } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { if(m_light_wallet) { @@ -2211,7 +2628,6 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // Update lw heights m_light_wallet_scanned_block_height = res.scanned_block_height; m_light_wallet_blockchain_height = res.blockchain_height; - m_local_bc_height = res.blockchain_height; // If new height - call new_block callback if(m_light_wallet_blockchain_height != prev_height) { @@ -2240,12 +2656,12 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; uint64_t blocks_start_height; - std::list<cryptonote::block_complete_entry> blocks; - std::vector<COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices; + std::vector<cryptonote::block_complete_entry> blocks; + std::vector<parsed_block> parsed_blocks; bool refreshed = false; // pull the first set of blocks - get_short_chain_history(short_chain_history); + get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); m_run.store(true, std::memory_order_relaxed); if (start_height > m_blockchain.size() || m_refresh_from_block_height > m_blockchain.size()) { if (!start_height) @@ -2254,7 +2670,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re fast_refresh(start_height, blocks_start_height, short_chain_history); // regenerate the history now that we've got a full set of hashes short_chain_history.clear(); - get_short_chain_history(short_chain_history); + get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); start_height = 0; // and then fall through to regular refresh processing } @@ -2262,55 +2678,102 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // If stop() is called during fast refresh we don't need to continue if(!m_run.load(std::memory_order_relaxed)) return; - pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); // always reset start_height to 0 to force short_chain_ history to be used on // subsequent pulls in this refresh. start_height = 0; + auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { + if (m_encrypt_keys_after_refresh) + { + encrypt_keys(*m_encrypt_keys_after_refresh); + m_encrypt_keys_after_refresh = boost::none; + } + }); + + bool first = true; while(m_run.load(std::memory_order_relaxed)) { try { // pull the next set of blocks while we're processing the current one uint64_t next_blocks_start_height; - std::list<cryptonote::block_complete_entry> next_blocks; - std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> next_o_indices; + std::vector<cryptonote::block_complete_entry> next_blocks; + std::vector<parsed_block> next_parsed_blocks; bool error = false; - if (blocks.empty()) + added_blocks = 0; + if (!first && blocks.empty()) { refreshed = false; break; } - tpool.submit(&waiter, [&]{pull_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, next_o_indices, error);}); + tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, parsed_blocks, next_blocks, next_parsed_blocks, error);}); - process_blocks(blocks_start_height, blocks, o_indices, added_blocks); - blocks_fetched += added_blocks; - waiter.wait(); - if(blocks_start_height == next_blocks_start_height) + if (!first) + { + try + { + process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks); + } + catch (const tools::error::out_of_hashchain_bounds_error&) + { + MINFO("Daemon claims next refresh block is out of hash chain bounds, resetting hash chain"); + uint64_t stop_height = m_blockchain.offset(); + std::vector<crypto::hash> tip(m_blockchain.size() - m_blockchain.offset()); + for (size_t i = m_blockchain.offset(); i < m_blockchain.size(); ++i) + tip[i - m_blockchain.offset()] = m_blockchain[i]; + cryptonote::block b; + generate_genesis(b); + m_blockchain.clear(); + m_blockchain.push_back(get_block_hash(b)); + short_chain_history.clear(); + get_short_chain_history(short_chain_history); + fast_refresh(stop_height, blocks_start_height, short_chain_history, true); + THROW_WALLET_EXCEPTION_IF(m_blockchain.size() != stop_height, error::wallet_internal_error, "Unexpected hashchain size"); + THROW_WALLET_EXCEPTION_IF(m_blockchain.offset() != 0, error::wallet_internal_error, "Unexpected hashchain offset"); + for (const auto &h: tip) + m_blockchain.push_back(h); + short_chain_history.clear(); + get_short_chain_history(short_chain_history); + start_height = stop_height; + throw std::runtime_error(""); // loop again + } + blocks_fetched += added_blocks; + } + waiter.wait(&tpool); + if(!first && blocks_start_height == next_blocks_start_height) { m_node_rpc_proxy.set_height(m_blockchain.size()); refreshed = true; break; } - // switch to the new blocks from the daemon - blocks_start_height = next_blocks_start_height; - blocks = next_blocks; - o_indices = next_o_indices; + first = false; // handle error from async fetching thread if (error) { throw std::runtime_error("proxy exception in refresh thread"); } + + // switch to the new blocks from the daemon + blocks_start_height = next_blocks_start_height; + blocks = std::move(next_blocks); + parsed_blocks = std::move(next_parsed_blocks); + } + catch (const tools::error::password_needed&) + { + blocks_fetched += added_blocks; + waiter.wait(&tpool); + throw; } catch (const std::exception&) { blocks_fetched += added_blocks; - waiter.wait(); + waiter.wait(&tpool); if(try_count < 3) { LOG_PRINT_L1("Another try pull_blocks (try_count=" << try_count << ")..."); + first = true; ++try_count; } else @@ -2334,14 +2797,16 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re LOG_PRINT_L1("Failed to check pending transactions"); } + m_first_refresh_done = true; + LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all())); } //---------------------------------------------------------------------------------------------------- -bool wallet2::refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok) +bool wallet2::refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok) { try { - refresh(0, blocks_fetched, received_money); + refresh(trusted_daemon, 0, blocks_fetched, received_money); ok = true; } catch (...) @@ -2351,7 +2816,7 @@ bool wallet2::refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok) return ok; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) +bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) { uint32_t rpc_version; boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); @@ -2385,6 +2850,7 @@ bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64 req.amounts.push_back(0); req.from_height = 0; req.cumulative = true; + req.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2413,8 +2879,8 @@ bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64 MWARNING("Failed to request output distribution: results are not for amount 0"); return false; } - start_height = res.distributions[0].start_height; - distribution = std::move(res.distributions[0].distribution); + start_height = res.distributions[0].data.start_height; + distribution = std::move(res.distributions[0].data.distribution); return true; } //---------------------------------------------------------------------------------------------------- @@ -2462,7 +2928,6 @@ void wallet2::detach_blockchain(uint64_t height) size_t blocks_detached = m_blockchain.size() - height; m_blockchain.crop(height); - m_local_bc_height -= blocks_detached; for (auto it = m_payments.begin(); it != m_payments.end(); ) { @@ -2486,6 +2951,8 @@ void wallet2::detach_blockchain(uint64_t height) bool wallet2::deinit() { m_is_initialized=false; + unlock_keys_file(); + m_account.deinit(); return true; } //---------------------------------------------------------------------------------------------------- @@ -2504,9 +2971,9 @@ bool wallet2::clear() m_scanned_pool_txs[0].clear(); m_scanned_pool_txs[1].clear(); m_address_book.clear(); - m_local_bc_height = 1; m_subaddresses.clear(); m_subaddress_labels.clear(); + m_multisig_rounds_passed = 0; return true; } @@ -2521,10 +2988,23 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable { std::string account_data; std::string multisig_signers; + std::string multisig_derivations; cryptonote::account_base account = m_account; + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + { + account.encrypt_viewkey(key); + account.decrypt_keys(key); + } + if (watch_only) account.forget_spend_key(); + + account.encrypt_keys(key); + bool r = epee::serialization::store_t_to_binary(account, account_data); CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet keys"); wallet2::keys_file_data keys_file_data = boost::value_initialized<wallet2::keys_file_data>(); @@ -2543,7 +3023,7 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable rapidjson::Value value2(rapidjson::kNumberType); - value2.SetInt(m_key_on_device?1:0); + value2.SetInt(m_key_device_type); json.AddMember("key_on_device", value2, json.GetAllocator()); value2.SetInt(watch_only ? 1 :0); // WTF ? JSON has different true and false types, and not boolean ?? @@ -2561,6 +3041,14 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig signers"); value.SetString(multisig_signers.c_str(), multisig_signers.length()); json.AddMember("multisig_signers", value, json.GetAllocator()); + + r = ::serialization::dump_binary(m_multisig_derivations, multisig_derivations); + CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig derivations"); + value.SetString(multisig_derivations.c_str(), multisig_derivations.length()); + json.AddMember("multisig_derivations", value, json.GetAllocator()); + + value2.SetUint(m_multisig_rounds_passed); + json.AddMember("multisig_rounds_passed", value2, json.GetAllocator()); } value2.SetInt(m_always_confirm_transfers ? 1 :0); @@ -2593,7 +3081,7 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetInt(m_confirm_non_default_ring_size ? 1 :0); json.AddMember("confirm_non_default_ring_size", value2, json.GetAllocator()); - value2.SetInt(m_ask_password ? 1 :0); + value2.SetInt(m_ask_password); json.AddMember("ask_password", value2, json.GetAllocator()); value2.SetUint(m_min_output_count); @@ -2632,12 +3120,21 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetUint(m_segregation_height); json.AddMember("segregation_height", value2, json.GetAllocator()); + value2.SetInt(m_ignore_fractional_outputs ? 1 : 0); + json.AddMember("ignore_fractional_outputs", value2, json.GetAllocator()); + value2.SetUint(m_subaddress_lookahead_major); json.AddMember("subaddress_lookahead_major", value2, json.GetAllocator()); value2.SetUint(m_subaddress_lookahead_minor); json.AddMember("subaddress_lookahead_minor", value2, json.GetAllocator()); + value2.SetUint(1); + json.AddMember("encrypted_secret_keys", value2, json.GetAllocator()); + + value.SetString(m_device_name.c_str(), m_device_name.size()); + json.AddMember("device_name", value, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -2645,22 +3142,52 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable account_data = buffer.GetString(); // Encrypt the entire JSON object. - crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key); + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string cipher; cipher.resize(account_data.size()); keys_file_data.iv = crypto::rand<crypto::chacha_iv>(); crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); keys_file_data.account_data = cipher; + unlock_keys_file(); std::string buf; r = ::serialization::dump_binary(keys_file_data, buf); r = r && epee::file_io_utils::save_string_to_file(keys_file_name, buf); //and never touch wallet_keys_file again, only read CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << keys_file_name); + lock_keys_file(); return true; } //---------------------------------------------------------------------------------------------------- +void wallet2::setup_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + + // re-encrypt, but keep viewkey unencrypted + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + { + m_account.encrypt_keys(key); + m_account.decrypt_viewkey(key); + } + + static_assert(HASH_SIZE == sizeof(crypto::chacha_key), "Mismatched sizes of hash and chacha key"); + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE+1>> cache_key_data; + memcpy(cache_key_data.data(), &key, HASH_SIZE); + cache_key_data[HASH_SIZE] = CACHE_KEY_TAIL; + cn_fast_hash(cache_key_data.data(), HASH_SIZE+1, (crypto::hash&)m_cache_key); + get_ringdb_key(); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password) +{ + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + decrypt_keys(original_password); + setup_keys(new_password); + rewrite(filename, new_password); + store(); +} +//---------------------------------------------------------------------------------------------------- /*! * \brief Load wallet information from wallet file. * \param keys_file_name Name of wallet file @@ -2671,6 +3198,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ rapidjson::Document json; wallet2::keys_file_data keys_file_data; std::string buf; + bool encrypted_secret_keys = false; bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); @@ -2678,7 +3206,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ r = ::serialization::parse_binary(buf, keys_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key); + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); @@ -2693,6 +3221,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + m_multisig_rounds_passed = 0; + m_multisig_derivations.clear(); m_always_confirm_transfers = false; m_print_ring_members = false; m_default_mixin = 0; @@ -2701,7 +3231,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_refresh_type = RefreshType::RefreshDefault; m_confirm_missing_payment_id = true; m_confirm_non_default_ring_size = true; - m_ask_password = true; + m_ask_password = AskPasswordToDecrypt; m_min_output_count = 0; m_min_output_value = 0; m_merge_destinations = false; @@ -2712,9 +3242,12 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_segregate_pre_fork_outputs = true; m_key_reuse_mitigation2 = true; m_segregation_height = 0; + m_ignore_fractional_outputs = true; m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR; m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; - m_key_on_device = false; + m_device_name = ""; + m_key_device_type = hw::device::device_type::SOFTWARE; + encrypted_secret_keys = false; } else if(json.IsObject()) { @@ -2733,8 +3266,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ if (json.HasMember("key_on_device")) { - GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, key_on_device, int, Int, false, false); - m_key_on_device = field_key_on_device; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, key_on_device, int, Int, false, hw::device::device_type::SOFTWARE); + m_key_device_type = static_cast<hw::device::device_type>(field_key_on_device); } GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, seed_language, std::string, String, false, std::string()); @@ -2748,6 +3281,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_multisig = field_multisig; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, multisig_threshold, unsigned int, Uint, m_multisig, 0); m_multisig_threshold = field_multisig_threshold; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, multisig_rounds_passed, unsigned int, Uint, false, 0); + m_multisig_rounds_passed = field_multisig_rounds_passed; if (m_multisig) { if (!json.HasMember("multisig_signers")) @@ -2768,6 +3303,24 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ LOG_ERROR("Field multisig_signers found in JSON, but failed to parse"); return false; } + + //previous version of multisig does not have this field + if (json.HasMember("multisig_derivations")) + { + if (!json["multisig_derivations"].IsString()) + { + LOG_ERROR("Field multisig_derivations found in JSON, but not String"); + return false; + } + const char *field_multisig_derivations = json["multisig_derivations"].GetString(); + std::string multisig_derivations = std::string(field_multisig_derivations, field_multisig_derivations + json["multisig_derivations"].GetStringLength()); + r = ::serialization::parse_binary(multisig_derivations, m_multisig_derivations); + if (!r) + { + LOG_ERROR("Field multisig_derivations found in JSON, but failed to parse"); + return false; + } + } } GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, always_confirm_transfers, int, Int, false, true); m_always_confirm_transfers = field_always_confirm_transfers; @@ -2808,7 +3361,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_confirm_missing_payment_id = field_confirm_missing_payment_id; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, confirm_non_default_ring_size, int, Int, false, true); m_confirm_non_default_ring_size = field_confirm_non_default_ring_size; - GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ask_password, int, Int, false, true); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ask_password, AskPasswordType, Int, false, AskPasswordToDecrypt); m_ask_password = field_ask_password; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, default_decimal_point, int, Int, false, CRYPTONOTE_DISPLAY_DECIMAL_POINT); cryptonote::set_default_decimal_point(field_default_decimal_point); @@ -2838,10 +3391,28 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_key_reuse_mitigation2 = field_key_reuse_mitigation2; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, segregation_height, int, Uint, false, 0); m_segregation_height = field_segregation_height; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ignore_fractional_outputs, int, Int, false, true); + m_ignore_fractional_outputs = field_ignore_fractional_outputs; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_major, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MAJOR); m_subaddress_lookahead_major = field_subaddress_lookahead_major; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_minor, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MINOR); m_subaddress_lookahead_minor = field_subaddress_lookahead_minor; + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, encrypted_secret_keys, uint32_t, Uint, false, false); + encrypted_secret_keys = field_encrypted_secret_keys; + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_name, std::string, String, false, std::string()); + if (m_device_name.empty()) + { + if (field_device_name_found) + { + m_device_name = field_device_name; + } + else + { + m_device_name = m_key_device_type == hw::device::device_type::LEDGER ? "Ledger" : "default"; + } + } } else { @@ -2850,20 +3421,58 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ } r = epee::serialization::load_t_from_binary(m_account, account_data); - if (r && m_key_on_device) { + THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); + if (m_key_device_type == hw::device::device_type::LEDGER || m_key_device_type == hw::device::device_type::TREZOR) { LOG_PRINT_L0("Account on device. Initing device..."); - hw::device &hwdev = hw::get_device("Ledger"); - hwdev.init(); - hwdev.connect(); + hw::device &hwdev = lookup_device(m_device_name); + THROW_WALLET_EXCEPTION_IF(!hwdev.set_name(m_device_name), error::wallet_internal_error, "Could not set device name " + m_device_name); + hwdev.set_network_type(m_nettype); + THROW_WALLET_EXCEPTION_IF(!hwdev.init(), error::wallet_internal_error, "Could not initialize the device " + m_device_name); + THROW_WALLET_EXCEPTION_IF(!hwdev.connect(), error::wallet_internal_error, "Could not connect to the device " + m_device_name); m_account.set_device(hwdev); + + account_public_address device_account_public_address; + THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. " + "Device address: " + cryptonote::get_account_address_as_str(m_nettype, false, device_account_public_address) + + ", wallet address: " + m_account.get_public_address_str(m_nettype)); LOG_PRINT_L0("Device inited..."); + } else if (key_on_device()) { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, "hardware device not supported"); + } + + if (r) + { + if (encrypted_secret_keys) + { + m_account.decrypt_keys(key); + } + else + { + // rewrite with encrypted keys, ignore errors + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + encrypt_keys(key); + bool saved_ret = store_keys(keys_file_name, password, m_watch_only); + if (!saved_ret) + { + // just moan a bit, but not fatal + MERROR("Error saving keys file with encrypted keys, not fatal"); + } + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + decrypt_keys(key); + m_keys_file_locker.reset(); + } } const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); - if(!m_watch_only && !m_multisig) + if(!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); + + if (r) + setup_keys(password); + return true; } @@ -2877,9 +3486,13 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const epee::wipeable_string& password) const +bool wallet2::verify_password(const epee::wipeable_string& password) { - return verify_password(m_keys_file, password, m_watch_only || m_multisig, m_account.get_device()); + // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). + unlock_keys_file(); + bool r = verify_password(m_keys_file, password, m_account.get_device().device_protocol() == hw::device::PROTOCOL_COLD || m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); + lock_keys_file(); + return r; } /*! @@ -2895,11 +3508,12 @@ bool wallet2::verify_password(const epee::wipeable_string& password) const * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev) +bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds) { rapidjson::Document json; wallet2::keys_file_data keys_file_data; std::string buf; + bool encrypted_secret_keys = false; bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); @@ -2907,7 +3521,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip r = ::serialization::parse_binary(buf, keys_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key); + crypto::generate_chacha_key(password.data(), password.size(), key, kdf_rounds); std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); @@ -2923,19 +3537,129 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip { account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() + json["key_data"].GetStringLength()); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, encrypted_secret_keys, uint32_t, Uint, false, false); + encrypted_secret_keys = field_encrypted_secret_keys; } cryptonote::account_base account_data_check; r = epee::serialization::load_t_from_binary(account_data_check, account_data); - const cryptonote::account_keys& keys = account_data_check.get_keys(); + if (encrypted_secret_keys) + account_data_check.decrypt_keys(key); + + const cryptonote::account_keys& keys = account_data_check.get_keys(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); if(!no_spend_key) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); return r; } +void wallet2::encrypt_keys(const crypto::chacha_key &key) +{ + m_account.encrypt_keys(key); + m_account.decrypt_viewkey(key); +} + +void wallet2::decrypt_keys(const crypto::chacha_key &key) +{ + m_account.encrypt_viewkey(key); + m_account.decrypt_keys(key); +} + +void wallet2::encrypt_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + encrypt_keys(key); +} + +void wallet2::decrypt_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + decrypt_keys(key); +} + +void wallet2::setup_new_blockchain() +{ + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); + add_subaddress_account(tr("Primary account")); +} + +void wallet2::create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file) +{ + if (!wallet_.empty()) + { + bool r = store_keys(m_keys_file, password, watch_only); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + if (create_address_file) + { + r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); + if(!r) MERROR("String with address text not saved"); + } + } +} + + +/*! + * \brief determine the key storage for the specified wallet file + * \param device_type (OUT) wallet backend as enumerated in hw::device::device_type + * \param keys_file_name Keys file to verify password for + * \param password Password to verify + * \return true if password correct, else false + * + * for verification only - determines key storage hardware + * + */ +bool wallet2::query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds) +{ + rapidjson::Document json; + wallet2::keys_file_data keys_file_data; + std::string buf; + bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); + THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); + + // Decrypt the contents + r = ::serialization::parse_binary(buf, keys_file_data); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, kdf_rounds); + std::string account_data; + account_data.resize(keys_file_data.account_data.size()); + crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + if (json.Parse(account_data.c_str()).HasParseError() || !json.IsObject()) + crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + + device_type = hw::device::device_type::SOFTWARE; + // The contents should be JSON if the wallet follows the new format. + if (json.Parse(account_data.c_str()).HasParseError()) + { + // old format before JSON wallet key file format + } + else + { + account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() + + json["key_data"].GetStringLength()); + + if (json.HasMember("key_on_device")) + { + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, key_on_device, int, Int, false, hw::device::device_type::SOFTWARE); + device_type = static_cast<hw::device::device_type>(field_key_on_device); + } + } + + cryptonote::account_base account_data_check; + + r = epee::serialization::load_t_from_binary(account_data_check, account_data); + if (!r) return false; + return true; +} + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -2944,7 +3668,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip * \param create_address_file Whether to create an address file */ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& password, - const std::string& multisig_data, bool create_address_file) + const epee::wipeable_string& multisig_data, bool create_address_file) { clear(); prepare_file_names(wallet_); @@ -3000,6 +3724,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& for (const auto &msk: multisig_keys) sc_add(skey.bytes, skey.bytes, rct::sk2rct(msk).bytes); THROW_WALLET_EXCEPTION_IF(!(rct::rct2sk(skey) == spend_secret_key), error::invalid_multisig_seed); + memwipe(&skey, sizeof(rct::key)); m_account.make_multisig(view_secret_key, spend_secret_key, spend_public_key, multisig_keys); m_account.finalize_multisig(spend_public_key); @@ -3009,24 +3734,11 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig = true; m_multisig_threshold = threshold; m_multisig_signers = multisig_signers; - m_key_on_device = false; - - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + m_key_device_type = hw::device::device_type::SOFTWARE; + setup_keys(password); - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - add_subaddress_account(tr("Primary account")); + create_keys_file(wallet_, false, password, m_nettype != MAINNET || create_address_file); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3062,29 +3774,17 @@ crypto::secret_key wallet2::generate(const std::string& wallet_, const epee::wip m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); - m_key_on_device = false; + m_key_device_type = hw::device::device_type::SOFTWARE; + setup_keys(password); // calculate a starting refresh height if(m_refresh_from_block_height == 0 && !recover){ m_refresh_from_block_height = estimate_blockchain_height(); } - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + create_keys_file(wallet_, false, password, m_nettype != MAINNET || create_address_file); - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3162,24 +3862,12 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); - m_key_on_device = false; + m_key_device_type = hw::device::device_type::SOFTWARE; + setup_keys(password); - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, true); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + create_keys_file(wallet_, true, password, m_nettype != MAINNET || create_address_file); - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3214,24 +3902,12 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); - m_key_on_device = false; + m_key_device_type = hw::device::device_type::SOFTWARE; + setup_keys(password); - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } + create_keys_file(wallet_, false, password, create_address_file); - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3243,7 +3919,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& * \param password Password of wallet file * \param device_name device string address */ -void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name) +void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name, bool create_address_file) { clear(); prepare_file_names(wallet_); @@ -3253,25 +3929,29 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(m_wallet_file, ignored_ec), error::file_exists, m_wallet_file); THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(m_keys_file, ignored_ec), error::file_exists, m_keys_file); } - m_key_on_device = true; - m_account.create_from_device(device_name); + + auto &hwdev = lookup_device(device_name); + hwdev.set_name(device_name); + hwdev.set_network_type(m_nettype); + + m_account.create_from_device(hwdev); + m_key_device_type = m_account.get_device().get_type(); m_account_public_address = m_account.get_keys().m_account_address; m_watch_only = false; m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + setup_keys(password); + m_device_name = device_name; - if (!wallet_.empty()) { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); + create_keys_file(wallet_, false, password, m_nettype != MAINNET || create_address_file); + if (m_subaddress_lookahead_major == SUBADDRESS_LOOKAHEAD_MAJOR && m_subaddress_lookahead_minor == SUBADDRESS_LOOKAHEAD_MINOR) + { + // the default lookahead setting (50:200) is clearly too much for hardware wallet + m_subaddress_lookahead_major = 5; + m_subaddress_lookahead_minor = 20; } - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) { store(); } @@ -3285,103 +3965,266 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, CHECK_AND_ASSERT_THROW_MES(!view_keys.empty(), "empty view keys"); CHECK_AND_ASSERT_THROW_MES(view_keys.size() == spend_keys.size(), "Mismatched view/spend key sizes"); CHECK_AND_ASSERT_THROW_MES(threshold > 1 && threshold <= spend_keys.size() + 1, "Invalid threshold"); - CHECK_AND_ASSERT_THROW_MES(threshold == spend_keys.size() || threshold == spend_keys.size() + 1, "Unsupported threshold case"); std::string extra_multisig_info; - crypto::hash hash; + std::vector<crypto::secret_key> multisig_keys; + rct::key spend_pkey = rct::identity(); + rct::key spend_skey; + std::vector<crypto::public_key> multisig_signers; - clear(); + // decrypt keys + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + { + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(password.data(), password.size(), chacha_key, m_kdf_rounds); + m_account.encrypt_viewkey(chacha_key); + m_account.decrypt_keys(chacha_key); + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this, chacha_key]() { m_account.encrypt_keys(chacha_key); m_account.decrypt_viewkey(chacha_key); }); + } + + // In common multisig scheme there are 4 types of key exchange rounds: + // 1. First round is exchange of view secret keys and public spend keys. + // 2. Middle round is exchange of derivations: Ki = b * Mj, where b - spend secret key, + // M - public multisig key (in first round it equals to public spend key), K - new public multisig key. + // 3. Secret spend establishment round sets your secret multisig keys as follows: kl = H(Ml), where M - is *your* public multisig key, + // k - secret multisig key used to sign transactions. k and M are sets of keys, of course. + // And secret spend key as the sum of all participant's secret multisig keys + // 4. Last round establishes multisig wallet's public spend key. Participants exchange their public multisig keys + // and calculate common spend public key as sum of all unique participants' public multisig keys. + // Note that N/N scheme has only first round. N-1/N has 2 rounds: first and last. Common M/N has all 4 rounds. + + // IMPORTANT: wallet's public spend key is not equal to secret_spend_key * G! + // Wallet's public spend key is the sum of unique public multisig keys of all participants. + // secret_spend_key * G = public signer key - MINFO("Creating spend key..."); - std::vector<crypto::secret_key> multisig_keys; - rct::key spend_pkey, spend_skey; if (threshold == spend_keys.size() + 1) { + // In N / N case we only need to do one round and calculate secret multisig keys and new secret spend key + MINFO("Creating spend key..."); + + // Calculates all multisig keys and spend key cryptonote::generate_multisig_N_N(get_account().get_keys(), spend_keys, multisig_keys, spend_skey, spend_pkey); + + // Our signer key is b * G, where b is secret spend key. + multisig_signers = spend_keys; + multisig_signers.push_back(get_multisig_signer_public_key(get_account().get_keys().m_spend_secret_key)); } - else if (threshold == spend_keys.size()) + else { - cryptonote::generate_multisig_N1_N(get_account().get_keys(), spend_keys, multisig_keys, spend_skey, spend_pkey); + // We just got public spend keys of all participants and deriving multisig keys (set of Mi = b * Bi). + // note that derivations are public keys as DH exchange suppose it to be + auto derivations = cryptonote::generate_multisig_derivations(get_account().get_keys(), spend_keys); - // We need an extra step, so we package all the composite public keys - // we know about, and make a signed string out of them - std::string data; - crypto::public_key signer; - CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(rct::rct2sk(spend_skey), signer), "Failed to derive public spend key"); - data += std::string((const char *)&signer, sizeof(crypto::public_key)); + spend_pkey = rct::identity(); + multisig_signers = std::vector<crypto::public_key>(spend_keys.size() + 1, crypto::null_pkey); - for (const auto &msk: multisig_keys) + if (threshold == spend_keys.size()) { - rct::key pmsk = rct::scalarmultBase(rct::sk2rct(msk)); - data += std::string((const char *)&pmsk, sizeof(crypto::public_key)); + // N - 1 / N case + + // We need an extra step, so we package all the composite public keys + // we know about, and make a signed string out of them + MINFO("Creating spend key..."); + + // Calculating set of our secret multisig keys as follows: mi = H(Mi), + // where mi - secret multisig key, Mi - others' participants public multisig key + multisig_keys = cryptonote::calculate_multisig_keys(derivations); + + // calculating current participant's spend secret key as sum of all secret multisig keys for current participant. + // IMPORTANT: participant's secret spend key is not an entire wallet's secret spend! + // Entire wallet's secret spend is sum of all unique secret multisig keys + // among all of participants and is not held by anyone! + spend_skey = rct::sk2rct(cryptonote::calculate_multisig_signer_key(multisig_keys)); + + // Preparing data for the last round to calculate common public spend key. The data contains public multisig keys. + extra_multisig_info = pack_multisignature_keys(MULTISIG_EXTRA_INFO_MAGIC, secret_keys_to_public_keys(multisig_keys), rct::rct2sk(spend_skey)); } + else + { + // M / N case + MINFO("Preparing keys for next exchange round..."); - data.resize(data.size() + sizeof(crypto::signature)); - crypto::cn_fast_hash(data.data(), data.size() - sizeof(signature), hash); - crypto::signature &signature = *(crypto::signature*)&data[data.size() - sizeof(crypto::signature)]; - crypto::generate_signature(hash, signer, rct::rct2sk(spend_skey), signature); + // Preparing data for middle round - packing new public multisig keys to exchage with others. + extra_multisig_info = pack_multisignature_keys(MULTISIG_EXTRA_INFO_MAGIC, derivations, m_account.get_keys().m_spend_secret_key); + spend_skey = rct::sk2rct(m_account.get_keys().m_spend_secret_key); - extra_multisig_info = std::string("MultisigxV1") + tools::base58::encode(data); - } - else - { - CHECK_AND_ASSERT_THROW_MES(false, "Unsupported threshold case"); + // Need to store middle keys to be able to proceed in case of wallet shutdown. + m_multisig_derivations = derivations; + } } - // the multisig view key is shared by all, make one all can derive + clear(); MINFO("Creating view key..."); crypto::secret_key view_skey = cryptonote::generate_multisig_view_secret_key(get_account().get_keys().m_view_secret_key, view_keys); MINFO("Creating multisig address..."); CHECK_AND_ASSERT_THROW_MES(m_account.make_multisig(view_skey, rct::rct2sk(spend_skey), rct::rct2pk(spend_pkey), multisig_keys), "Failed to create multisig wallet due to bad keys"); + memwipe(&spend_skey, sizeof(rct::key)); m_account_public_address = m_account.get_keys().m_account_address; m_watch_only = false; m_multisig = true; + m_key_device_type = hw::device::device_type::SOFTWARE; m_multisig_threshold = threshold; - m_key_on_device = false; + m_multisig_signers = multisig_signers; + ++m_multisig_rounds_passed; - if (threshold == spend_keys.size() + 1) + // re-encrypt keys + keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); + + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); + + setup_new_blockchain(); + + if (!m_wallet_file.empty()) + store(); + + return extra_multisig_info; +} + +std::string wallet2::exchange_multisig_keys(const epee::wipeable_string &password, + const std::vector<std::string> &info) +{ + THROW_WALLET_EXCEPTION_IF(info.empty(), + error::wallet_internal_error, "Empty multisig info"); + + if (info[0].substr(0, MULTISIG_EXTRA_INFO_MAGIC.size()) != MULTISIG_EXTRA_INFO_MAGIC) { - m_multisig_signers = spend_keys; - m_multisig_signers.push_back(get_multisig_signer_public_key()); + THROW_WALLET_EXCEPTION_IF(false, + error::wallet_internal_error, "Unsupported info string"); } - else + + std::vector<crypto::public_key> signers; + std::unordered_set<crypto::public_key> pkeys; + + THROW_WALLET_EXCEPTION_IF(!unpack_extra_multisig_info(info, signers, pkeys), + error::wallet_internal_error, "Bad extra multisig info"); + + return exchange_multisig_keys(password, pkeys, signers); +} + +std::string wallet2::exchange_multisig_keys(const epee::wipeable_string &password, + std::unordered_set<crypto::public_key> derivations, + std::vector<crypto::public_key> signers) +{ + CHECK_AND_ASSERT_THROW_MES(!derivations.empty(), "empty pkeys"); + CHECK_AND_ASSERT_THROW_MES(!signers.empty(), "empty signers"); + + bool ready = false; + CHECK_AND_ASSERT_THROW_MES(multisig(&ready), "The wallet is not multisig"); + CHECK_AND_ASSERT_THROW_MES(!ready, "Multisig wallet creation process has already been finished"); + + // keys are decrypted + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) { - m_multisig_signers = std::vector<crypto::public_key>(spend_keys.size() + 1, crypto::null_pkey); + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(password.data(), password.size(), chacha_key, m_kdf_rounds); + m_account.encrypt_viewkey(chacha_key); + m_account.decrypt_keys(chacha_key); + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this, chacha_key]() { m_account.encrypt_keys(chacha_key); m_account.decrypt_viewkey(chacha_key); }); } - if (!m_wallet_file.empty()) + if (m_multisig_rounds_passed == multisig_rounds_required(m_multisig_signers.size(), m_multisig_threshold) - 1) { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + // the last round is passed and we have to calculate spend public key + // add ours if not included + crypto::public_key local_signer = get_multisig_signer_public_key(); - if (boost::filesystem::exists(m_wallet_file + ".address.txt")) + if (std::find(signers.begin(), signers.end(), local_signer) == signers.end()) { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); + signers.push_back(local_signer); + for (const auto &msk: get_account().get_multisig_keys()) + { + derivations.insert(rct::rct2pk(rct::scalarmultBase(rct::sk2rct(msk)))); + } + } + + CHECK_AND_ASSERT_THROW_MES(signers.size() == m_multisig_signers.size(), "Bad signers size"); + + // Summing all of unique public multisig keys to calculate common public spend key + crypto::public_key spend_public_key = cryptonote::generate_multisig_M_N_spend_public_key(std::vector<crypto::public_key>(derivations.begin(), derivations.end())); + m_account_public_address.m_spend_public_key = spend_public_key; + m_account.finalize_multisig(spend_public_key); + + m_multisig_signers = signers; + std::sort(m_multisig_signers.begin(), m_multisig_signers.end(), [](const crypto::public_key &e0, const crypto::public_key &e1){ return memcmp(&e0, &e1, sizeof(e0)); }); + + ++m_multisig_rounds_passed; + m_multisig_derivations.clear(); + + // keys are encrypted again + keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); + + if (!m_wallet_file.empty()) + { + bool r = store_keys(m_keys_file, password, false); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + if (boost::filesystem::exists(m_wallet_file + ".address.txt")) + { + r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); + if(!r) MERROR("String with address text not saved"); + } } + + m_subaddresses.clear(); + m_subaddress_labels.clear(); + add_subaddress_account(tr("Primary account")); + + if (!m_wallet_file.empty()) + store(); + + return {}; } - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - add_subaddress_account(tr("Primary account")); + // Below are either middle or secret spend key establishment rounds - if (!m_wallet_file.empty()) - store(); + for (const auto& key: m_multisig_derivations) + derivations.erase(key); + + // Deriving multisig keys (set of Mi = b * Bi) according to DH from other participants' multisig keys. + auto new_derivations = cryptonote::generate_multisig_derivations(get_account().get_keys(), std::vector<crypto::public_key>(derivations.begin(), derivations.end())); + + std::string extra_multisig_info; + if (m_multisig_rounds_passed == multisig_rounds_required(m_multisig_signers.size(), m_multisig_threshold) - 2) // next round is last + { + // Next round is last therefore we are performing secret spend establishment round as described above. + MINFO("Creating spend key..."); + + // Calculating our secret multisig keys by hashing our public multisig keys. + auto multisig_keys = cryptonote::calculate_multisig_keys(std::vector<crypto::public_key>(new_derivations.begin(), new_derivations.end())); + // And summing it to get personal secret spend key + crypto::secret_key spend_skey = cryptonote::calculate_multisig_signer_key(multisig_keys); + m_account.make_multisig(m_account.get_keys().m_view_secret_key, spend_skey, rct::rct2pk(rct::identity()), multisig_keys); + + // Packing public multisig keys to exchange with others and calculate common public spend key in the last round + extra_multisig_info = pack_multisignature_keys(MULTISIG_EXTRA_INFO_MAGIC, secret_keys_to_public_keys(multisig_keys), spend_skey); + } + else + { + // This is just middle round + MINFO("Preparing keys for next exchange round..."); + extra_multisig_info = pack_multisignature_keys(MULTISIG_EXTRA_INFO_MAGIC, new_derivations, m_account.get_keys().m_spend_secret_key); + m_multisig_derivations = new_derivations; + } + + ++m_multisig_rounds_passed; + + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); return extra_multisig_info; } -std::string wallet2::make_multisig(const epee::wipeable_string &password, - const std::vector<std::string> &info, - uint32_t threshold) +void wallet2::unpack_multisig_info(const std::vector<std::string>& info, + std::vector<crypto::public_key> &public_keys, + std::vector<crypto::secret_key> &secret_keys) const { // parse all multisig info - std::vector<crypto::secret_key> secret_keys(info.size()); - std::vector<crypto::public_key> public_keys(info.size()); + public_keys.resize(info.size()); + secret_keys.resize(info.size()); for (size_t i = 0; i < info.size(); ++i) { THROW_WALLET_EXCEPTION_IF(!verify_multisig_info(info[i], secret_keys[i], public_keys[i]), @@ -3425,71 +4268,51 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, "Found local spend public key, but not local view secret key - something very weird"); } } +} +std::string wallet2::make_multisig(const epee::wipeable_string &password, + const std::vector<std::string> &info, + uint32_t threshold) +{ + std::vector<crypto::secret_key> secret_keys(info.size()); + std::vector<crypto::public_key> public_keys(info.size()); + unpack_multisig_info(info, public_keys, secret_keys); return make_multisig(password, secret_keys, public_keys, threshold); } bool wallet2::finalize_multisig(const epee::wipeable_string &password, std::unordered_set<crypto::public_key> pkeys, std::vector<crypto::public_key> signers) { - CHECK_AND_ASSERT_THROW_MES(!pkeys.empty(), "empty pkeys"); - - // add ours if not included - crypto::public_key local_signer; - CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(get_account().get_keys().m_spend_secret_key, local_signer), - "Failed to derive public spend key"); - if (std::find(signers.begin(), signers.end(), local_signer) == signers.end()) - { - signers.push_back(local_signer); - for (const auto &msk: get_account().get_multisig_keys()) - { - pkeys.insert(rct::rct2pk(rct::scalarmultBase(rct::sk2rct(msk)))); - } - } - - CHECK_AND_ASSERT_THROW_MES(signers.size() == m_multisig_signers.size(), "Bad signers size"); - - crypto::public_key spend_public_key = cryptonote::generate_multisig_N1_N_spend_public_key(std::vector<crypto::public_key>(pkeys.begin(), pkeys.end())); - m_account_public_address.m_spend_public_key = spend_public_key; - m_account.finalize_multisig(spend_public_key); - - m_multisig_signers = signers; - std::sort(m_multisig_signers.begin(), m_multisig_signers.end(), [](const crypto::public_key &e0, const crypto::public_key &e1){ return memcmp(&e0, &e1, sizeof(e0)); }); + exchange_multisig_keys(password, pkeys, signers); + return true; +} - if (!m_wallet_file.empty()) +bool wallet2::unpack_extra_multisig_info(const std::vector<std::string>& info, + std::vector<crypto::public_key> &signers, + std::unordered_set<crypto::public_key> &pkeys) const +{ + // parse all multisig info + signers.resize(info.size(), crypto::null_pkey); + for (size_t i = 0; i < info.size(); ++i) { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - - if (boost::filesystem::exists(m_wallet_file + ".address.txt")) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } + if (!verify_extra_multisig_info(info[i], pkeys, signers[i])) + { + return false; + } } - m_subaddresses.clear(); - m_subaddress_labels.clear(); - add_subaddress_account(tr("Primary account")); - - if (!m_wallet_file.empty()) - store(); - return true; } bool wallet2::finalize_multisig(const epee::wipeable_string &password, const std::vector<std::string> &info) { - // parse all multisig info std::unordered_set<crypto::public_key> public_keys; - std::vector<crypto::public_key> signers(info.size(), crypto::null_pkey); - for (size_t i = 0; i < info.size(); ++i) + std::vector<crypto::public_key> signers; + if (!unpack_extra_multisig_info(info, signers, public_keys)) { - if (!verify_extra_multisig_info(info[i], public_keys, signers[i])) - { - MERROR("Bad multisig info"); - return false; - } + MERROR("Bad multisig info"); + return false; } + return finalize_multisig(password, public_keys, signers); } @@ -3552,14 +4375,13 @@ bool wallet2::verify_multisig_info(const std::string &data, crypto::secret_key & bool wallet2::verify_extra_multisig_info(const std::string &data, std::unordered_set<crypto::public_key> &pkeys, crypto::public_key &signer) { - const size_t header_len = strlen("MultisigxV1"); - if (data.size() < header_len || data.substr(0, header_len) != "MultisigxV1") + if (data.size() < MULTISIG_EXTRA_INFO_MAGIC.size() || data.substr(0, MULTISIG_EXTRA_INFO_MAGIC.size()) != MULTISIG_EXTRA_INFO_MAGIC) { MERROR("Multisig info header check error"); return false; } std::string decoded; - if (!tools::base58::decode(data.substr(header_len), decoded)) + if (!tools::base58::decode(data.substr(MULTISIG_EXTRA_INFO_MAGIC.size()), decoded)) { MERROR("Multisig info decoding error"); return false; @@ -3622,6 +4444,14 @@ bool wallet2::has_multisig_partial_key_images() const return false; } +bool wallet2::has_unknown_key_images() const +{ + for (const auto &td: m_transfers) + if (!td.m_key_image_known) + return true; + return false; +} + /*! * \brief Rewrites to the wallet file for wallet upgrade (doesn't generate key, assumes it's already there) * \param wallet_name Name of wallet file (should exist) @@ -3755,7 +4585,12 @@ bool wallet2::check_connection(uint32_t *version, uint32_t timeout) bool wallet2::generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const { hw::device &hwdev = m_account.get_device(); - return hwdev.generate_chacha_key(m_account.get_keys(), key); + return hwdev.generate_chacha_key(m_account.get_keys(), key, m_kdf_rounds); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::generate_chacha_key_from_password(const epee::wipeable_string &pass, crypto::chacha_key &key) const +{ + crypto::generate_chacha_key(pass.data(), pass.size(), key, m_kdf_rounds); } //---------------------------------------------------------------------------------------------------- void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password) @@ -3766,12 +4601,19 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass boost::system::error_code e; bool exists = boost::filesystem::exists(m_keys_file, e); THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); + lock_keys_file(); + THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); + // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). + unlock_keys_file(); if (!load_keys(m_keys_file, password)) { THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); } LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); + lock_keys_file(); + + wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, password); //keys loaded ok! //try to load wallet file. but even if we failed, it is not big problem @@ -3784,7 +4626,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass { wallet2::cache_file_data cache_file_data; std::string buf; - bool r = epee::file_io_utils::load_file_to_string(m_wallet_file, buf); + bool r = epee::file_io_utils::load_file_to_string(m_wallet_file, buf, std::numeric_limits<size_t>::max()); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); // try to read it as an encrypted cache @@ -3794,11 +4636,9 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass r = ::serialization::parse_binary(buf, cache_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"'); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); std::string cache_data; cache_data.resize(cache_file_data.cache_data.size()); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cache_data[0]); try { std::stringstream iss; @@ -3806,11 +4646,13 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass boost::archive::portable_binary_iarchive ar(iss); ar >> *this; } - catch (...) + catch(...) { - crypto::chacha8(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); - try - { + // try with previous scheme: direct from keys + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + try { std::stringstream iss; iss << cache_data; boost::archive::portable_binary_iarchive ar(iss); @@ -3818,13 +4660,24 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass } catch (...) { - LOG_PRINT_L0("Failed to open portable binary, trying unportable"); - boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); - std::stringstream iss; - iss.str(""); - iss << cache_data; - boost::archive::binary_iarchive ar(iss); - ar >> *this; + crypto::chacha8(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + try + { + std::stringstream iss; + iss << cache_data; + boost::archive::portable_binary_iarchive ar(iss); + ar >> *this; + } + catch (...) + { + LOG_PRINT_L0("Failed to open portable binary, trying unportable"); + boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + std::stringstream iss; + iss.str(""); + iss << cache_data; + boost::archive::binary_iarchive ar(iss); + ar >> *this; + } } } } @@ -3861,6 +4714,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass if (m_blockchain.empty()) { m_blockchain.push_back(genesis_hash); + m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } else { @@ -3872,8 +4726,6 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass if (get_num_subaddress_accounts() == 0) add_subaddress_account(tr("Primary account")); - m_local_bc_height = m_blockchain.size(); - try { find_and_save_rings(false); @@ -3977,12 +4829,10 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas wallet2::cache_file_data cache_file_data = boost::value_initialized<wallet2::cache_file_data>(); cache_file_data.cache_data = oss.str(); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); std::string cipher; cipher.resize(cache_file_data.cache_data.size()); cache_file_data.iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cipher[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cipher[0]); cache_file_data.cache_data = cipher; const std::string new_file = same_file ? m_wallet_file + ".new" : path; @@ -4248,15 +5098,10 @@ void wallet2::rescan_blockchain(bool refresh) { clear(); - cryptonote::block genesis; - generate_genesis(genesis); - crypto::hash genesis_hash = get_block_hash(genesis); - m_blockchain.push_back(genesis_hash); - add_subaddress_account(tr("Primary account")); - m_local_bc_height = 1; + setup_new_blockchain(); if (refresh) - this->refresh(); + this->refresh(false); } //---------------------------------------------------------------------------------------------------- bool wallet2::is_transfer_unlocked(const transfer_details& td) const @@ -4269,7 +5114,7 @@ bool wallet2::is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) if(!is_tx_spendtime_unlocked(unlock_time, block_height)) return false; - if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > m_local_bc_height) + if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > get_blockchain_current_height()) return false; return true; @@ -4280,7 +5125,7 @@ bool wallet2::is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_heig if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { //interpret as block index - if(m_local_bc_height-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + if(get_blockchain_current_height()-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) return true; else return false; @@ -4290,7 +5135,7 @@ bool wallet2::is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_heig uint64_t current_time = static_cast<uint64_t>(time(NULL)); // XXX: this needs to be fast, so we'd need to get the starting heights // from the daemon to be correct once voting kicks in - uint64_t v2height = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? (uint64_t)-1/*TODO*/ : 1009827; + uint64_t v2height = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? 32000 : 1009827; uint64_t leeway = block_height < v2height ? CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V1 : CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V2; if(current_time + leeway >= unlock_time) return true; @@ -4440,7 +5285,7 @@ size_t wallet2::pop_best_value(std::vector<size_t> &unused_indices, const std::v // returns: // direct return: amount of money found // modified reference: selected_transfers, a list of iterators/indices of input sources -uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const +uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const { uint64_t found_money = 0; selected_transfers.reserve(unused_transfers_indices.size()); @@ -4483,69 +5328,6 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo } //---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon) -{ - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx, trusted_daemon); -} -//---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon) -{ - cryptonote::transaction tx; - pending_tx ptx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx, trusted_daemon); -} - -namespace { -// split_amounts(vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -// -// split amount for each dst in dsts into num_splits parts -// and make num_splits new vector<crypt...> instances to hold these new amounts -std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts( - std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -{ - std::vector<std::vector<cryptonote::tx_destination_entry>> retVal; - - if (num_splits <= 1) - { - retVal.push_back(dsts); - return retVal; - } - - // for each split required - for (size_t i=0; i < num_splits; i++) - { - std::vector<cryptonote::tx_destination_entry> new_dsts; - - // for each destination - for (size_t j=0; j < dsts.size(); j++) - { - cryptonote::tx_destination_entry de; - uint64_t amount; - - amount = dsts[j].amount; - amount = amount / num_splits; - - // if last split, add remainder - if (i + 1 == num_splits) - { - amount += dsts[j].amount % num_splits; - } - - de.addr = dsts[j].addr; - de.amount = amount; - - new_dsts.push_back(de); - } - - retVal.push_back(new_dsts); - } - - return retVal; -} -} // anonymous namespace -//---------------------------------------------------------------------------------------------------- crypto::hash wallet2::get_payment_id(const pending_tx &ptx) const { std::vector<tx_extra_field> tx_extra_fields; @@ -4665,6 +5447,15 @@ void wallet2::commit_tx(std::vector<pending_tx>& ptx_vector) bool wallet2::save_tx(const std::vector<pending_tx>& ptx_vector, const std::string &filename) const { LOG_PRINT_L0("saving " << ptx_vector.size() << " transactions"); + std::string ciphertext = dump_tx_to_str(ptx_vector); + if (ciphertext.empty()) + return false; + return epee::file_io_utils::save_string_to_file(filename, ciphertext); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) const +{ + LOG_PRINT_L0("saving " << ptx_vector.size() << " transactions"); unsigned_tx_set txs; for (auto &tx: ptx_vector) { @@ -4684,11 +5475,11 @@ bool wallet2::save_tx(const std::vector<pending_tx>& ptx_vector, const std::stri } catch (...) { - return false; + return std::string(); } LOG_PRINT_L2("Saving unsigned tx data: " << oss.str()); std::string ciphertext = encrypt_with_view_secret_key(oss.str()); - return epee::file_io_utils::save_string_to_file(filename, std::string(UNSIGNED_TX_PREFIX) + ciphertext); + return std::string(UNSIGNED_TX_PREFIX) + ciphertext; } //---------------------------------------------------------------------------------------------------- bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx_set &exported_txs) const @@ -4706,10 +5497,17 @@ bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx LOG_PRINT_L0("Failed to load from " << unsigned_filename); return false; } + + return parse_unsigned_tx_from_str(s, exported_txs); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const +{ + std::string s = unsigned_tx_st; const size_t magiclen = strlen(UNSIGNED_TX_PREFIX) - 1; if (strncmp(s.c_str(), UNSIGNED_TX_PREFIX, magiclen)) { - LOG_PRINT_L0("Bad magic from " << unsigned_filename); + LOG_PRINT_L0("Bad magic from unsigned tx"); return false; } s = s.substr(magiclen); @@ -4725,7 +5523,7 @@ bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx } catch (...) { - LOG_PRINT_L0("Failed to parse data from " << unsigned_filename); + LOG_PRINT_L0("Failed to parse data from unsigned tx"); return false; } } @@ -4742,19 +5540,19 @@ bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx } catch (...) { - LOG_PRINT_L0("Failed to parse data from " << unsigned_filename); + LOG_PRINT_L0("Failed to parse data from unsigned tx"); return false; } } catch (const std::exception &e) { - LOG_PRINT_L0("Failed to decrypt " << unsigned_filename << ": " << e.what()); + LOG_PRINT_L0("Failed to decrypt unsigned tx: " << e.what()); return false; } } else { - LOG_PRINT_L0("Unsupported version in " << unsigned_filename); + LOG_PRINT_L0("Unsupported version in unsigned tx"); return false; } LOG_PRINT_L1("Loaded tx unsigned data from binary: " << exported_txs.txes.size() << " transactions"); @@ -4775,14 +5573,12 @@ bool wallet2::sign_tx(const std::string &unsigned_filename, const std::string &s } return sign_tx(exported_txs, signed_filename, txs, export_raw); } - //---------------------------------------------------------------------------------------------------- -bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_filename, std::vector<wallet2::pending_tx> &txs, bool export_raw) +bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &txs, signed_tx_set &signed_txes) { import_outputs(exported_txs.transfers); // sign the transactions - signed_tx_set signed_txes; for (size_t n = 0; n < exported_txs.txes.size(); ++n) { tools::wallet2::tx_construction_data &sd = exported_txs.txes[n]; @@ -4790,11 +5586,15 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - bool bulletproof = sd.use_rct && !ptx.tx.rct_signatures.p.bulletproofs.empty(); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofPaddedBulletproof; + } crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, bulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, range_proof_type, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -4833,6 +5633,10 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f ptx.construction_data = sd; txs.push_back(ptx); + + // add tx keys only to ptx + txs.back().tx_key = tx_key; + txs.back().additional_tx_keys = additional_tx_keys; } // add key images @@ -4844,20 +5648,21 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f signed_txes.key_images[i] = m_transfers[i].m_key_image; } - // save as binary - std::ostringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - try - { - ar << signed_txes; - } - catch(...) + return true; +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_filename, std::vector<wallet2::pending_tx> &txs, bool export_raw) +{ + // sign the transactions + signed_tx_set signed_txes; + std::string ciphertext = sign_tx_dump_to_str(exported_txs, txs, signed_txes); + if (ciphertext.empty()) { + LOG_PRINT_L0("Failed to sign unsigned_tx_set"); return false; } - LOG_PRINT_L3("Saving signed tx data (with encryption): " << oss.str()); - std::string ciphertext = encrypt_with_view_secret_key(oss.str()); - if (!epee::file_io_utils::save_string_to_file(signed_filename, std::string(SIGNED_TX_PREFIX) + ciphertext)) + + if (!epee::file_io_utils::save_string_to_file(signed_filename, ciphertext)) { LOG_PRINT_L0("Failed to save file to " << signed_filename); return false; @@ -4879,6 +5684,32 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f return true; } //---------------------------------------------------------------------------------------------------- +std::string wallet2::sign_tx_dump_to_str(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &ptx, signed_tx_set &signed_txes) +{ + // sign the transactions + bool r = sign_tx(exported_txs, ptx, signed_txes); + if (!r) + { + LOG_PRINT_L0("Failed to sign unsigned_tx_set"); + return std::string(); + } + + // save as binary + std::ostringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + try + { + ar << signed_txes; + } + catch(...) + { + return std::string(); + } + LOG_PRINT_L3("Saving signed tx data (with encryption): " << oss.str()); + std::string ciphertext = encrypt_with_view_secret_key(oss.str()); + return std::string(SIGNED_TX_PREFIX) + ciphertext; +} +//---------------------------------------------------------------------------------------------------- bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func) { std::string s; @@ -4896,10 +5727,20 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal LOG_PRINT_L0("Failed to load from " << signed_filename); return false; } + + return parse_tx_from_str(s, ptx, accept_func); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func) +{ + std::string s = signed_tx_st; + boost::system::error_code errcode; + signed_tx_set signed_txs; + const size_t magiclen = strlen(SIGNED_TX_PREFIX) - 1; if (strncmp(s.c_str(), SIGNED_TX_PREFIX, magiclen)) { - LOG_PRINT_L0("Bad magic from " << signed_filename); + LOG_PRINT_L0("Bad magic from signed transaction"); return false; } s = s.substr(magiclen); @@ -4915,7 +5756,7 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal } catch (...) { - LOG_PRINT_L0("Failed to parse data from " << signed_filename); + LOG_PRINT_L0("Failed to parse data from signed transaction"); return false; } } @@ -4932,23 +5773,23 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal } catch (...) { - LOG_PRINT_L0("Failed to parse decrypted data from " << signed_filename); + LOG_PRINT_L0("Failed to parse decrypted data from signed transaction"); return false; } } catch (const std::exception &e) { - LOG_PRINT_L0("Failed to decrypt " << signed_filename << ": " << e.what()); + LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what()); return false; } } else { - LOG_PRINT_L0("Unsupported version in " << signed_filename); + LOG_PRINT_L0("Unsupported version in signed transaction"); return false; } LOG_PRINT_L0("Loaded signed tx data from binary: " << signed_txs.ptx.size() << " transactions"); - for (auto &ptx: signed_txs.ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(ptx.tx)); + for (auto &c_ptx: signed_txs.ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(c_ptx.tx)); if (accept_func && !accept_func(signed_txs)) { @@ -4957,22 +5798,8 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal } // import key images - if (signed_txs.key_images.size() > m_transfers.size()) - { - LOG_PRINT_L1("More key images returned that we know outputs for"); - return false; - } - for (size_t i = 0; i < signed_txs.key_images.size(); ++i) - { - transfer_details &td = m_transfers[i]; - if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != signed_txs.key_images[i]) - LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); - td.m_key_image = signed_txs.key_images[i]; - m_key_images[m_transfers[i].m_key_image] = i; - td.m_key_image_known = true; - td.m_key_image_partial = false; - m_pub_keys[m_transfers[i].get_public_key()] = i; - } + bool r = import_key_images(signed_txs.key_images); + if (!r) return false; ptx = signed_txs.ptx; @@ -5170,8 +5997,15 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - const bool bulletproof = sd.use_rct && (ptx.tx.rct_signatures.type == rct::RCTTypeFullBulletproof || ptx.tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof); - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, bulletproof, &msout, false); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofBulletproof; + for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) + if (proof.V.size() > 1) + range_proof_type = rct::RangeProofPaddedBulletproof; + } + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -5271,9 +6105,18 @@ bool wallet2::sign_multisig_tx_from_file(const std::string &filename, std::vecto //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const { - static const uint64_t old_multipliers[3] = {1, 2, 3}; - static const uint64_t new_multipliers[3] = {1, 20, 166}; - static const uint64_t newer_multipliers[4] = {1, 4, 20, 166}; + static const struct + { + size_t count; + uint64_t multipliers[4]; + } + multipliers[] = + { + { 3, {1, 2, 3} }, + { 3, {1, 20, 166} }, + { 4, {1, 4, 20, 166} }, + { 4, {1, 5, 25, 1000} }, + }; if (fee_algorithm == -1) fee_algorithm = get_fee_algorithm(); @@ -5289,47 +6132,68 @@ uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const priority = 1; } + THROW_WALLET_EXCEPTION_IF(fee_algorithm < 0 || fee_algorithm > 3, error::invalid_priority); + // 1 to 3/4 are allowed as priorities - uint32_t max_priority = (fee_algorithm >= 2) ? 4 : 3; + const uint32_t max_priority = multipliers[fee_algorithm].count; if (priority >= 1 && priority <= max_priority) { - switch (fee_algorithm) - { - case 0: return old_multipliers[priority-1]; - case 1: return new_multipliers[priority-1]; - case 2: return newer_multipliers[priority-1]; - default: THROW_WALLET_EXCEPTION_IF (true, error::invalid_priority); - } + return multipliers[fee_algorithm].multipliers[priority-1]; } THROW_WALLET_EXCEPTION_IF (false, error::invalid_priority); return 1; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_dynamic_per_kb_fee_estimate() const +uint64_t wallet2::get_dynamic_base_fee_estimate() const { uint64_t fee; - boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_per_kb_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); + boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_base_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); if (!result) return fee; - LOG_PRINT_L1("Failed to query per kB fee, using " << print_money(FEE_PER_KB)); - return FEE_PER_KB; + const uint64_t base_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE) ? FEE_PER_BYTE : FEE_PER_KB; + LOG_PRINT_L1("Failed to query base fee, using " << print_money(base_fee)); + return base_fee; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_per_kb_fee() const +uint64_t wallet2::get_base_fee() const { if(m_light_wallet) - return m_light_wallet_per_kb_fee; + { + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + return m_light_wallet_per_kb_fee / 1024; + else + return m_light_wallet_per_kb_fee; + } bool use_dyn_fee = use_fork_rules(HF_VERSION_DYNAMIC_FEE, -720 * 1); if (!use_dyn_fee) return FEE_PER_KB; - return get_dynamic_per_kb_fee_estimate(); + return get_dynamic_base_fee_estimate(); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::get_fee_quantization_mask() const +{ + if(m_light_wallet) + { + return 1; // TODO + } + bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + if (!use_per_byte_fee) + return 1; + + uint64_t fee_quantization_mask; + boost::optional<std::string> result = m_node_rpc_proxy.get_fee_quantization_mask(fee_quantization_mask); + if (result) + return 1; + return fee_quantization_mask; } //---------------------------------------------------------------------------------------------------- int wallet2::get_fee_algorithm() const { - // changes at v3 and v5 + // changes at v3, v5, v8 + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0)) + return 3; if (use_fork_rules(5, 0)) return 2; if (use_fork_rules(3, -720 * 14)) @@ -5337,19 +6201,39 @@ int wallet2::get_fee_algorithm() const return 0; } //------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_min_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + if (use_fork_rules(7, 10)) + return 7; + if (use_fork_rules(6, 10)) + return 5; + if (use_fork_rules(2, 10)) + return 3; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_max_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ uint64_t wallet2::adjust_mixin(uint64_t mixin) const { - if (mixin < 6 && use_fork_rules(7, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 7, using 7"); - mixin = 6; - } - else if (mixin < 4 && use_fork_rules(6, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 6, using 5"); - mixin = 4; + const uint64_t min_ring_size = get_min_ring_size(); + if (mixin + 1 < min_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too low, using " << min_ring_size); + mixin = min_ring_size-1; } - else if (mixin < 2 && use_fork_rules(2, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 2, using 3"); - mixin = 2; + const uint64_t max_ring_size = get_max_ring_size(); + if (max_ring_size && mixin + 1 > max_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too high, using " << max_ring_size); + mixin = max_ring_size-1; } return mixin; } @@ -5361,7 +6245,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) try { // check if there's a backlog in the tx pool - const double fee_level = get_fee_multiplier(1) * get_per_kb_fee() * (12/(double)13) / (double)1024; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + const uint64_t base_fee = get_base_fee(); + const uint64_t fee_multiplier = get_fee_multiplier(1); + const double fee_level = fee_multiplier * base_fee * (use_per_byte_fee ? 1 : (12/(double)13 / (double)1024)); const std::vector<std::pair<uint64_t, uint64_t>> blocks = estimate_backlog({std::make_pair(fee_level, fee_level)}); if (blocks.size() != 1) { @@ -5375,15 +6262,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) } // get the current full reward zone - cryptonote::COMMAND_RPC_GET_INFO::request getinfo_req = AUTO_VAL_INIT(getinfo_req); - cryptonote::COMMAND_RPC_GET_INFO::response getinfo_res = AUTO_VAL_INIT(getinfo_res); - m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", getinfo_req, getinfo_res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); - THROW_WALLET_EXCEPTION_IF(getinfo_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); - THROW_WALLET_EXCEPTION_IF(getinfo_res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - const uint64_t full_reward_zone = getinfo_res.block_size_limit / 2; + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); + throw_on_rpc_response_error(result, "get_info"); + const uint64_t full_reward_zone = block_weight_limit / 2; // get the last N block headers and sum the block sizes const size_t N = 10; @@ -5397,7 +6279,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) m_daemon_rpc_mutex.lock(); getbh_req.start_height = m_blockchain.size() - N; getbh_req.end_height = m_blockchain.size() - 1; - r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblockheadersrange"); THROW_WALLET_EXCEPTION_IF(getbh_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblockheadersrange"); @@ -5407,14 +6289,14 @@ uint32_t wallet2::adjust_priority(uint32_t priority) MERROR("Bad blockheaders size"); return priority; } - size_t block_size_sum = 0; + size_t block_weight_sum = 0; for (const cryptonote::block_header_response &i : getbh_res.headers) { - block_size_sum += i.block_size; + block_weight_sum += i.block_weight; } // estimate how 'full' the last N blocks are - const size_t P = 100 * block_size_sum / (N * full_reward_zone); + const size_t P = 100 * block_weight_sum / (N * full_reward_zone); MINFO((boost::format("The last %d blocks fill roughly %d%% of the full reward zone.") % N % P).str()); if (P > 80) { @@ -5432,115 +6314,6 @@ uint32_t wallet2::adjust_priority(uint32_t priority) return priority; } //---------------------------------------------------------------------------------------------------- -// separated the call(s) to wallet2::transfer into their own function -// -// this function will make multiple calls to wallet2::transfer if multiple -// transactions will be required -std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) -{ - const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true, trusted_daemon); - - const uint64_t fee_per_kb = get_per_kb_fee(); - const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); - - // failsafe split attempt counter - size_t attempt_count = 0; - - for(attempt_count = 1; ;attempt_count++) - { - size_t num_tx = 0.5 + pow(1.7,attempt_count-1); - - auto split_values = split_amounts(dsts, num_tx); - - // Throw if split_amounts comes back with a vector of size different than it should - if (split_values.size() != num_tx) - { - throw std::runtime_error("Splitting transactions returned a number of potential tx not equal to what was requested"); - } - - std::vector<pending_tx> ptx_vector; - try - { - // for each new destination vector (i.e. for each new tx) - for (auto & dst_vector : split_values) - { - cryptonote::transaction tx; - pending_tx ptx; - - // loop until fee is met without increasing tx size to next KB boundary. - const size_t estimated_tx_size = estimate_tx_size(false, unused_transfers_indices.size(), fake_outs_count, dst_vector.size(), extra.size(), false); - uint64_t needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); - do - { - transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx, trusted_daemon); - auto txBlob = t_serializable_object_to_blob(ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - } while (ptx.fee < needed_fee); - - ptx_vector.push_back(ptx); - - // mark transfers to be used as "spent" - for(size_t idx: ptx.selected_transfers) - { - set_spent(idx, 0); - } - } - - // if we made it this far, we've selected our transactions. committing them will mark them spent, - // so this is a failsafe in case they don't go through - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - - } - - // if we made it this far, we're OK to actually send the transactions - return ptx_vector; - - } - // only catch this here, other exceptions need to pass through to the calling function - catch (const tools::error::tx_too_big& e) - { - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - if (attempt_count >= MAX_SPLIT_ATTEMPTS) - { - throw; - } - } - catch (...) - { - // in case of some other exception, make sure any tx in queue are marked unspent again - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - throw; - } - } -} - bool wallet2::set_ring_database(const std::string &filename) { m_ring_database = filename; @@ -5564,6 +6337,31 @@ bool wallet2::set_ring_database(const std::string &filename) return true; } +crypto::chacha_key wallet2::get_ringdb_key() +{ + if (!m_ringdb_key) + { + MINFO("caching ringdb key"); + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + m_ringdb_key = key; + } + return *m_ringdb_key; +} + +void wallet2::register_devices(){ + hw::trezor::register_all(); +} + +hw::device& wallet2::lookup_device(const std::string & device_descriptor){ + if (!m_devices_registered){ + m_devices_registered = true; + register_devices(); + } + + return hw::get_device(device_descriptor); +} + bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -5574,9 +6372,7 @@ bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transac bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return add_rings(key, tx); } + try { return add_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5584,9 +6380,7 @@ bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) { if (!m_ringdb) return false; - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return m_ringdb->remove_rings(key, tx); } + try { return m_ringdb->remove_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5623,10 +6417,7 @@ bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto:: bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - - try { return get_ring(key, key_image, outs); } + try { return get_ring(get_ringdb_key(), key_image, outs); } catch (const std::exception &e) { return false; } } @@ -5635,10 +6426,7 @@ bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uin if (!m_ringdb) return false; - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - - try { return m_ringdb->set_ring(key, key_image, outs, relative); } + try { return m_ringdb->set_ring(get_ringdb_key(), key_image, outs, relative); } catch (const std::exception &e) { return false; } } @@ -5666,9 +6454,6 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - // get those transactions from the daemon static const size_t SLICE_SIZE = 200; for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) @@ -5705,7 +6490,7 @@ bool wallet2::find_and_save_rings(bool force) crypto::hash tx_hash, tx_prefix_hash; THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); - THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring"); + THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); } } @@ -5714,7 +6499,7 @@ bool wallet2::find_and_save_rings(bool force) return true; } -bool wallet2::blackball_output(const crypto::public_key &output) +bool wallet2::blackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -5722,7 +6507,7 @@ bool wallet2::blackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add) +bool wallet2::set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add) { if (!m_ringdb) return false; @@ -5731,14 +6516,13 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out bool ret = true; if (!add) ret &= m_ringdb->clear_blackballs(); - for (const auto &output: outputs) - ret &= m_ringdb->blackball(output); + ret &= m_ringdb->blackball(outputs); return ret; } catch (const std::exception &e) { return false; } } -bool wallet2::unblackball_output(const crypto::public_key &output) +bool wallet2::unblackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -5746,7 +6530,7 @@ bool wallet2::unblackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::is_output_blackballed(const crypto::public_key &output) const +bool wallet2::is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const { if (!m_ringdb) return false; @@ -5754,6 +6538,33 @@ bool wallet2::is_output_blackballed(const crypto::public_key &output) const catch (const std::exception &e) { return false; } } +bool wallet2::lock_keys_file() +{ + if (m_keys_file_locker) + { + MDEBUG(m_keys_file << " is already locked."); + return false; + } + m_keys_file_locker.reset(new tools::file_locker(m_keys_file)); + return true; +} + +bool wallet2::unlock_keys_file() +{ + if (!m_keys_file_locker) + { + MDEBUG(m_keys_file << " is already unlocked."); + return false; + } + m_keys_file_locker.reset(); + return true; +} + +bool wallet2::is_keys_file_locked() const +{ + return m_keys_file_locker->locked(); +} + bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const { if (!unlocked) // don't add locked outs @@ -5764,8 +6575,8 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; - if (is_output_blackballed(output_public_key)) // don't add blackballed outputs - return false; +// if (is_output_blackballed(output_public_key)) // don't add blackballed outputs +// return false; outs.back().push_back(item); return true; } @@ -5884,9 +6695,6 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> return; } - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - if (fake_outputs_count > 0) { uint64_t segregation_fork_height = get_segregation_fork_height(); @@ -5897,22 +6705,48 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> bool is_shortly_after_segregation_fork = height >= segregation_fork_height && height < segregation_fork_height + SEGREGATION_FORK_VICINITY; bool is_after_segregation_fork = height >= segregation_fork_height; + // if we have at least one rct out, get the distribution, or fall back to the previous system + uint64_t rct_start_height; + std::vector<uint64_t> rct_offsets; + bool has_rct = false; + uint64_t max_rct_index = 0; + for (size_t idx: selected_transfers) + if (m_transfers[idx].is_rct()) + { + has_rct = true; + max_rct_index = std::max(max_rct_index, m_transfers[idx].m_global_output_index); + } + const bool has_rct_distribution = has_rct && get_rct_distribution(rct_start_height, rct_offsets); + if (has_rct_distribution) + { + // check we're clear enough of rct start, to avoid corner cases below + THROW_WALLET_EXCEPTION_IF(rct_offsets.size() <= CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, + error::get_output_distribution, "Not enough rct outputs"); + THROW_WALLET_EXCEPTION_IF(rct_offsets.back() <= max_rct_index, + error::get_output_distribution, "Daemon reports suspicious number of rct outputs"); + } + // get histogram for the amounts we need cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); - m_daemon_rpc_mutex.lock(); + // request histogram for all outputs, except 0 if we have the rct distribution for(size_t idx: selected_transfers) - req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); - std::sort(req_t.amounts.begin(), req_t.amounts.end()); - auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); - req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); - req_t.unlocked = true; - req_t.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); - THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + if (!m_transfers[idx].is_rct() || !has_rct_distribution) + req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); + if (!req_t.amounts.empty()) + { + std::sort(req_t.amounts.begin(), req_t.amounts.end()); + auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); + req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); + req_t.unlocked = true; + req_t.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + } // if we want to segregate fake outs pre or post fork, get distribution std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> segregation_limit; @@ -5928,6 +6762,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> req_t.from_height = std::max<uint64_t>(segregation_fork_height, RECENT_OUTPUT_BLOCKS) - RECENT_OUTPUT_BLOCKS; req_t.to_height = segregation_fork_height + 1; req_t.cumulative = true; + req_t.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout * 1000); m_daemon_rpc_mutex.unlock(); @@ -5944,13 +6779,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> { if (d.amount == amount) { - THROW_WALLET_EXCEPTION_IF(d.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(d.data.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.data.start_height >= d.data.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.data.start_height >= d.data.distribution.size(), error::get_output_distribution, "Distribution size too small"); THROW_WALLET_EXCEPTION_IF(segregation_fork_height <= RECENT_OUTPUT_BLOCKS, error::wallet_internal_error, "Fork height too low"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.start_height, error::get_output_distribution, "Bad start height"); - uint64_t till_fork = d.distribution[segregation_fork_height - d.start_height]; - uint64_t recent = till_fork - d.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height]; + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.data.start_height, error::get_output_distribution, "Bad start height"); + uint64_t till_fork = d.data.distribution[segregation_fork_height - d.data.start_height]; + uint64_t recent = till_fork - d.data.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.data.start_height]; segregation_limit[amount] = std::make_pair(till_fork, recent); found = true; break; @@ -5968,6 +6803,55 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> COMMAND_RPC_GET_OUTPUTS_BIN::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_OUTPUTS_BIN::response daemon_resp = AUTO_VAL_INIT(daemon_resp); + struct gamma_engine + { + typedef uint64_t result_type; + static constexpr result_type min() { return 0; } + static constexpr result_type max() { return std::numeric_limits<result_type>::max(); } + result_type operator()() { return crypto::rand<result_type>(); } + } engine; + static const double shape = 19.28/*16.94*/; + //static const double shape = m_testnet ? 17.02 : 17.28; + static const double scale = 1/1.61; + std::gamma_distribution<double> gamma(shape, scale); + auto pick_gamma = [&]() + { + double x = gamma(engine); + x = exp(x); + uint64_t block_offset = x / DIFFICULTY_TARGET_V2; // this assumes constant target over the whole rct range + if (block_offset >= rct_offsets.size() - 1) + return std::numeric_limits<uint64_t>::max(); // bad pick + block_offset = rct_offsets.size() - 2 - block_offset; + THROW_WALLET_EXCEPTION_IF(block_offset >= rct_offsets.size() - 1, error::wallet_internal_error, "Bad offset calculation"); + THROW_WALLET_EXCEPTION_IF(rct_offsets[block_offset + 1] < rct_offsets[block_offset], + error::get_output_distribution, "Decreasing offsets in rct distribution: " + + std::to_string(block_offset) + ": " + std::to_string(rct_offsets[block_offset]) + ", " + + std::to_string(block_offset + 1) + ": " + std::to_string(rct_offsets[block_offset + 1])); + uint64_t first_block_offset = block_offset, last_block_offset = block_offset; + for (size_t half_window = 0; half_window < GAMMA_PICK_HALF_WINDOW; ++half_window) + { + // end when we have a non empty block + uint64_t cum0 = first_block_offset > 0 ? rct_offsets[first_block_offset] - rct_offsets[first_block_offset - 1] : rct_offsets[0]; + if (cum0 > 1) + break; + uint64_t cum1 = last_block_offset > 0 ? rct_offsets[last_block_offset] - rct_offsets[last_block_offset - 1] : rct_offsets[0]; + if (cum1 > 1) + break; + if (first_block_offset == 0 && last_block_offset >= rct_offsets.size() - 2) + break; + // expand up to bounds + if (first_block_offset > 0) + --first_block_offset; + if (last_block_offset < rct_offsets.size() - 1) + ++last_block_offset; + } + const uint64_t n_rct = rct_offsets[last_block_offset] - (first_block_offset == 0 ? 0 : rct_offsets[first_block_offset - 1]); + if (n_rct == 0) + return rct_offsets[block_offset] ? rct_offsets[block_offset] - 1 : 0; + MDEBUG("Picking 1/" << n_rct << " in " << (last_block_offset - first_block_offset + 1) << " blocks centered around " << block_offset); + return rct_offsets[first_block_offset] + crypto::rand<uint64_t>() % n_rct; + }; + size_t num_selected_transfers = 0; for(size_t idx: selected_transfers) { @@ -5978,6 +6862,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // request more for rct in base recent (locked) coinbases are picked, since they're locked for longer size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0); size_t start = req.outputs.size(); + bool use_histogram = amount != 0 || !has_rct_distribution; const bool output_is_pre_fork = td.m_block_height < segregation_fork_height; uint64_t num_outs = 0, num_recent_outs = 0; @@ -6033,26 +6918,41 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> num_post_fork_outs = num_outs - segregation_limit[amount].first; } - LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); - THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, - "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); - THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, - "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + if (use_histogram) + { + LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); + THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, + "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); + THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, + "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + } + else + { + // the base offset of the first rct output in the first unlocked block (or the one to be if there's none) + num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; + LOG_PRINT_L1("" << num_outs << " unlocked rct outputs"); + THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, + "histogram reports no unlocked rct outputs, not even ours"); + } - // how many fake outs to draw on a pre-fork triangular distribution + // how many fake outs to draw on a pre-fork distribution size_t pre_fork_outputs_count = requested_outputs_count * pre_fork_num_out_ratio; size_t post_fork_outputs_count = requested_outputs_count * post_fork_num_out_ratio; // how many fake outs to draw otherwise size_t normal_output_count = requested_outputs_count - pre_fork_outputs_count - post_fork_outputs_count; - // X% of those outs are to be taken from recent outputs - size_t recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; - if (recent_outputs_count == 0) - recent_outputs_count = 1; // ensure we have at least one, if possible - if (recent_outputs_count > num_recent_outs) - recent_outputs_count = num_recent_outs; - if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) - --recent_outputs_count; // if the real out is recent, pick one less recent fake out + size_t recent_outputs_count = 0; + if (use_histogram) + { + // X% of those outs are to be taken from recent outputs + recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; + if (recent_outputs_count == 0) + recent_outputs_count = 1; // ensure we have at least one, if possible + if (recent_outputs_count > num_recent_outs) + recent_outputs_count = num_recent_outs; + if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) + --recent_outputs_count; // if the real out is recent, pick one less recent fake out + } LOG_PRINT_L1("Fake output makeup: " << requested_outputs_count << " requested: " << recent_outputs_count << " recent, " << pre_fork_outputs_count << " pre-fork, " << post_fork_outputs_count << " post-fork, " << (requested_outputs_count - recent_outputs_count - pre_fork_outputs_count - post_fork_outputs_count) << " full-chain"); @@ -6064,7 +6964,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { MINFO("This output has a known ring, reusing (size " << ring.size() << ")"); THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error, @@ -6119,12 +7019,26 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount)); } + std::unordered_map<const char*, std::set<uint64_t>> picks; + // while we still need more mixins + uint64_t num_usable_outs = num_outs; + bool allow_blackballed = false; while (num_found < requested_outputs_count) { // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - break; + if (seen_indices.size() == num_usable_outs) + { + // there is a first pass which rejects blackballed outputs, then a second pass + // which allows them if we don't have enough non blackballed outputs to reach + // the required amount of outputs (since consensus does not care about blackballed + // outputs, we still need to reach the minimum ring size) + if (allow_blackballed) + break; + MINFO("Not enough output not marked as spent, we'll allow outputs marked as spent"); + allow_blackballed = true; + num_usable_outs = num_outs; + } // get a random output index from the DB. If we've already seen it, // return to the top of the loop and try again, otherwise add it to the @@ -6132,7 +7046,26 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> uint64_t i; const char *type = ""; - if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with + if (amount == 0 && has_rct_distribution) + { + // gamma distribution + if (num_found -1 < recent_outputs_count + pre_fork_outputs_count) + { + do i = pick_gamma(); while (i >= segregation_limit[amount].first); + type = "pre-fork gamma"; + } + else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count + post_fork_outputs_count) + { + do i = pick_gamma(); while (i < segregation_limit[amount].first || i >= num_outs); + type = "post-fork gamma"; + } + else + { + do i = pick_gamma(); while (i >= num_outs); + type = "gamma"; + } + } + else if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with { // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); @@ -6179,12 +7112,30 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (seen_indices.count(i)) continue; + if (!allow_blackballed && is_output_blackballed(std::make_pair(amount, i))) // don't add blackballed outputs + { + --num_usable_outs; + continue; + } seen_indices.emplace(i); - LOG_PRINT_L2("picking " << i << " as " << type); + picks[type].insert(i); req.outputs.push_back({amount, i}); ++num_found; } + + for (const auto &pick: picks) + MDEBUG("picking " << pick.first << " outputs: " << + boost::join(pick.second | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + + // if we had enough unusable outputs, we might fall off here and still + // have too few outputs, so we stuff with one to keep counts good, and + // we'll error out later + while (num_found < requested_outputs_count) + { + req.outputs.push_back({amount, 0}); + ++num_found; + } } // sort the subsection, to ensure the daemon doesn't know which output is ours @@ -6192,16 +7143,23 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> [](const get_outputs_out &a, const get_outputs_out &b) { return a.index < b.index; }); } - for (auto i: req.outputs) - LOG_PRINT_L1("asking for output " << i.index << " for " << print_money(i.amount)); + if (ELPP->vRegistry()->allowed(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY)) + { + std::map<uint64_t, std::set<uint64_t>> outs; + for (const auto &i: req.outputs) + outs[i.amount].insert(i.index); + for (const auto &o: outs) + MDEBUG("asking for outputs with amount " << print_money(o.first) << ": " << + boost::join(o.second | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + } // get the keys for those m_daemon_rpc_mutex.lock(); - r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, daemon_resp.status); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); @@ -6256,7 +7214,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { for (uint64_t out: ring) { @@ -6335,7 +7293,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer: starting with fee " << print_money (needed_money)); @@ -6437,10 +7395,10 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBulletproof, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); std::string key_images; bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool @@ -6474,6 +7432,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = false; + ptx.construction_data.use_bulletproofs = false; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -6485,13 +7444,13 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type) { using namespace cryptonote; // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer_selected_rct: starting with fee " << print_money (needed_money)); LOG_PRINT_L2("selected transfers: " << strjoin(selected_transfers, " ")); @@ -6641,10 +7600,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); auto sources_copy = sources; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, bulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, range_proof_type, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); // work out the permutation done on sources std::vector<size_t> ins_order; @@ -6678,7 +7637,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry THROW_WALLET_EXCEPTION_IF(selected_transfers.size() != sources.size(), error::wallet_internal_error, "mismatched selected_transfers and sources sixes"); for(size_t idx: selected_transfers) { - cryptonote::tx_source_entry& src = sources[src_idx]; + cryptonote::tx_source_entry& src = sources_copy[src_idx]; src.multisig_kLRki = get_multisig_composite_kLRki(idx, multisig_signers[signer_index], used_L, new_used_L); ++src_idx; } @@ -6686,10 +7645,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry LOG_PRINT_L2("Creating supplementary multisig transaction"); cryptonote::transaction ms_tx; auto sources_copy_copy = sources_copy; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, bulletproof, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, range_proof_type, &msout, false); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_prefix_hash(ms_tx) != prefix_hash, error::wallet_internal_error, "Multisig txes do not share prefix"); multisig_sigs.push_back({ms_tx.rct_signatures, multisig_signers[signer_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); @@ -6729,6 +7688,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = true; + ptx.construction_data.use_bulletproofs = !tx.rct_signatures.p.bulletproofs.empty(); ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -7155,6 +8115,7 @@ void wallet2::light_wallet_get_address_txs() payment.m_block_height = t.height; payment.m_unlock_time = t.unlock_time; payment.m_timestamp = t.timestamp; + payment.m_coinbase = t.coinbase; if (t.mempool) { if (std::find(unconfirmed_payments_txs.begin(), unconfirmed_payments_txs.end(), tx_hash) == unconfirmed_payments_txs.end()) { @@ -7335,7 +8296,7 @@ bool wallet2::light_wallet_key_image_is_ours(const crypto::key_image& key_image, // This system allows for sending (almost) the entire balance, since it does // not generate spurious change in all txes, thus decreasing the instantaneous // usable balance. -std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -7355,9 +8316,12 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; + uint64_t needed_fee; std::vector<std::vector<tools::wallet2::get_outs_entry>> outs; + TX() : weight(0), needed_fee(0) {} + void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) { @@ -7384,12 +8348,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<TX> txes; bool adding_fee; // true if new outputs go towards fee, rather than destinations uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); const bool use_rct = use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); @@ -7420,6 +8387,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // early out if we know we can't make it anyway // we could also check for being within FEE_PER_KB, but if the fee calculation // ever changes, this might be missed, so let this go through + const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)); uint64_t balance_subtotal = 0; uint64_t unlocked_balance_subtotal = 0; for (uint32_t index_minor : subaddr_indices) @@ -7427,21 +8395,33 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp balance_subtotal += balance_per_subaddr[index_minor]; unlocked_balance_subtotal += unlocked_balance_per_subaddr[index_minor]; } - THROW_WALLET_EXCEPTION_IF(needed_money > balance_subtotal, error::not_enough_money, + THROW_WALLET_EXCEPTION_IF(needed_money + min_fee > balance_subtotal, error::not_enough_money, balance_subtotal, needed_money, 0); // first check overall balance is enough, then unlocked one, so we throw distinct exceptions - THROW_WALLET_EXCEPTION_IF(needed_money > unlocked_balance_subtotal, error::not_enough_unlocked_money, + THROW_WALLET_EXCEPTION_IF(needed_money + min_fee > unlocked_balance_subtotal, error::not_enough_unlocked_money, unlocked_balance_subtotal, needed_money, 0); for (uint32_t i : subaddr_indices) LOG_PRINT_L2("Candidate subaddress index for spending: " << i); + // determine threshold for fractional amount + const size_t tx_weight_one_ring = estimate_tx_weight(use_rct, 1, fake_outs_count, 2, 0, bulletproof); + const size_t tx_weight_two_rings = estimate_tx_weight(use_rct, 2, fake_outs_count, 2, 0, bulletproof); + THROW_WALLET_EXCEPTION_IF(tx_weight_one_ring > tx_weight_two_rings, error::wallet_internal_error, "Estimated tx weight with 1 input is larger than with 2 inputs!"); + const size_t tx_weight_per_ring = tx_weight_two_rings - tx_weight_one_ring; + const uint64_t fractional_threshold = (fee_multiplier * base_fee * tx_weight_per_ring) / (use_per_byte_fee ? 1 : 1024); + // gather all dust and non-dust outputs belonging to specified subaddresses size_t num_nondust_outputs = 0; size_t num_dust_outputs = 0; for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; + if (m_ignore_fractional_outputs && td.amount() < fractional_threshold) + { + MDEBUG("Ignoring output " << i << " of amount " << print_money(td.amount()) << " which is below threshold " << print_money(fractional_threshold)); + continue; + } if (!td.m_spent && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { const uint32_t index_minor = td.m_subaddr_index.minor; @@ -7475,12 +8455,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } } - // shuffle & sort output indices + // sort output indices { - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(unused_transfers_indices_per_subaddr.begin(), unused_transfers_indices_per_subaddr.end(), g); - std::shuffle(unused_dust_indices_per_subaddr.begin(), unused_dust_indices_per_subaddr.end(), g); auto sort_predicate = [&unlocked_balance_per_subaddr] (const std::pair<uint32_t, std::vector<size_t>>& x, const std::pair<uint32_t, std::vector<size_t>>& y) { return unlocked_balance_per_subaddr[x.first] > unlocked_balance_per_subaddr[y.first]; @@ -7522,7 +8498,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp { // this is used to build a tx that's 1 or 2 inputs, and 2 outputs, which // will get us a known fee. - uint64_t estimated_fee = calculate_fee(fee_per_kb, estimate_rct_tx_size(2, fake_outs_count, 2, extra.size(), bulletproof), fee_multiplier); + uint64_t estimated_fee = estimate_fee(use_per_byte_fee, use_rct, 2, fake_outs_count, 2, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); preferred_inputs = pick_preferred_rct_inputs(needed_money + estimated_fee, subaddr_account, subaddr_indices); if (!preferred_inputs.empty()) { @@ -7564,7 +8540,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while ((!dsts.empty() && dsts[0].amount > 0) || adding_fee || !preferred_inputs.empty() || should_pick_a_second_output(use_rct, txes.back().selected_transfers.size(), *unused_transfers_indices, *unused_dust_indices)) { TX &tx = txes.back(); - LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size()); + LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size() << ", tx.dsts.size() " << tx.dsts.size()); LOG_PRINT_L2("unused_transfers_indices: " << strjoin(*unused_transfers_indices, " ")); LOG_PRINT_L2("unused_dust_indices: " << strjoin(*unused_dust_indices, " ")); LOG_PRINT_L2("dsts size " << dsts.size() << ", first " << (dsts.empty() ? "-" : cryptonote::print_money(dsts[0].amount))); @@ -7627,7 +8603,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) + while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << @@ -7639,7 +8615,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp ++original_output_index; } - if (available_amount > 0 && !dsts.empty() && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) { + if (available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); @@ -7651,7 +8627,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); + << upper_transaction_weight_limit); bool try_tx = false; // if we have preferred picks, but haven't yet used all of them, continue if (preferred_inputs.empty()) @@ -7663,8 +8639,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - try_tx = dsts.empty() || (estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + try_tx = dsts.empty() || (estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); } } @@ -7672,8 +8648,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); @@ -7690,14 +8665,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.selected_transfers.size() << " inputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); available_for_fee = test_ptx.fee + test_ptx.change_dts.amount + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0); - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0) @@ -7733,23 +8708,24 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while (needed_fee > test_ptx.fee) { if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; + tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; accumulated_change += test_ptx.change_dts.amount; adding_fee = false; @@ -7801,18 +8777,18 @@ skip_tx: fake_outs_count, /* CONST size_t fake_outputs_count, */ tx.outs, /* MOD std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, */ unlock_time, /* CONST uint64_t unlock_time, */ - needed_fee, /* CONST uint64_t fee, */ + tx.needed_fee, /* CONST uint64_t fee, */ extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ - bulletproof); + range_proof_type); } else { transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, - needed_fee, + tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), @@ -7822,7 +8798,7 @@ skip_tx: auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -7833,7 +8809,7 @@ skip_tx: for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -7843,7 +8819,7 @@ skip_tx: return ptx_vector; } -std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -7894,10 +8870,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, outputs, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -7915,10 +8891,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypt break; } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, outputs, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -7931,18 +8907,24 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; + uint64_t needed_fee; std::vector<std::vector<get_outs_entry>> outs; + + TX() : weight(0), needed_fee(0) {} }; std::vector<TX> txes; uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); std::vector<std::vector<get_outs_entry>> outs; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const uint64_t fee_per_kb = get_per_kb_fee(); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); LOG_PRINT_L2("Starting with " << unused_transfers_indices.size() << " non-dust outputs and " << unused_dust_indices.size() << " dust outputs"); @@ -7964,7 +8946,25 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // get a random unspent output and use it to pay next chunk. We try to alternate // dust and non dust to ensure we never get with only dust, from which we might // get a tx that can't pay for itself - size_t idx = unused_transfers_indices.empty() ? pop_best_value(unused_dust_indices, tx.selected_transfers) : unused_dust_indices.empty() ? pop_best_value(unused_transfers_indices, tx.selected_transfers) : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_per_kb * fee_multiplier * (upper_transaction_size_limit + 1023) / 1024) ? pop_best_value(unused_dust_indices, tx.selected_transfers) : pop_best_value(unused_transfers_indices, tx.selected_transfers); + uint64_t fee_dust_threshold; + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + { + const uint64_t estimated_tx_weight_with_one_extra_output = estimate_tx_weight(use_rct, tx.selected_transfers.size() + 1, fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + fee_dust_threshold = calculate_fee_from_weight(base_fee, estimated_tx_weight_with_one_extra_output, fee_multiplier, fee_quantization_mask); + } + else + { + fee_dust_threshold = base_fee * fee_multiplier * (upper_transaction_weight_limit + 1023) / 1024; + } + + size_t idx = + unused_transfers_indices.empty() + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : unused_dust_indices.empty() + ? pop_best_value(unused_transfers_indices, tx.selected_transfers) + : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_dust_threshold) + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : pop_best_value(unused_transfers_indices, tx.selected_transfers); const transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Picking output " << idx << ", amount " << print_money(td.amount())); @@ -7979,57 +8979,79 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 1, extra.size(), bulletproof); - bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + << upper_transaction_weight_limit); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 2, extra.size(), bulletproof); + bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); if (try_tx) { cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); - tx.dsts.push_back(tx_destination_entry(1, address, is_subaddress)); + // add N - 1 outputs for correct initial fee estimation + for (size_t i = 0; i < ((outputs > 1) ? outputs - 1 : outputs); ++i) + tx.dsts.push_back(tx_destination_entry(1, address, is_subaddress)); LOG_PRINT_L2("Trying to create a tx now, with " << tx.dsts.size() << " destinations and " << tx.selected_transfers.size() << " outputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - available_for_fee = test_ptx.fee + test_ptx.dests[0].amount + test_ptx.change_dts.amount; - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + available_for_fee = test_ptx.fee + test_ptx.change_dts.amount; + for (auto &dt: test_ptx.dests) + available_for_fee += dt.amount; + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); + // add last output, missed for fee estimation + if (outputs > 1) + tx.dsts.push_back(tx_destination_entry(1, address, is_subaddress)); + THROW_WALLET_EXCEPTION_IF(needed_fee > available_for_fee, error::wallet_internal_error, "Transaction cannot pay for itself"); do { LOG_PRINT_L2("We made a tx, adjusting fee and saving it"); - tx.dsts[0].amount = available_for_fee - needed_fee; + // distribute total transferred amount between outputs + uint64_t amount_transferred = available_for_fee - needed_fee; + uint64_t dt_amount = amount_transferred / outputs; + // residue is distributed as one atomic unit per output until it reaches zero + uint64_t residue = amount_transferred % outputs; + for (auto &dt: tx.dsts) + { + uint64_t dt_residue = 0; + if (residue > 0) + { + dt_residue = 1; + residue -= 1; + } + dt.amount = dt_amount + dt_residue; + } if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } while (needed_fee > test_ptx.fee); - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; + tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; accumulated_change += test_ptx.change_dts.amount; if (!unused_transfers_indices.empty() || !unused_dust_indices.empty()) @@ -8050,16 +9072,16 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton cryptonote::transaction test_tx; pending_tx test_ptx; if (use_rct) { - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, + test_tx, test_ptx, range_proof_type); } else { - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, needed_fee, extra, + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); } auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -8070,7 +9092,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8080,6 +9102,62 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton return ptx_vector; } //---------------------------------------------------------------------------------------------------- +void wallet2::cold_tx_aux_import(const std::vector<pending_tx> & ptx, const std::vector<std::string> & tx_device_aux) +{ + CHECK_AND_ASSERT_THROW_MES(ptx.size() == tx_device_aux.size(), "TX aux has invalid size"); + for (size_t i = 0; i < ptx.size(); ++i){ + crypto::hash txid; + txid = get_transaction_hash(ptx[i].tx); + set_tx_device_aux(txid, tx_device_aux[i]); + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux) +{ + auto & hwdev = get_account().get_device(); + if (!hwdev.has_tx_cold_sign()){ + throw std::invalid_argument("Device does not support cold sign protocol"); + } + + unsigned_tx_set txs; + for (auto &tx: ptx_vector) + { + txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); + } + txs.transfers = m_transfers; + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + hw::tx_aux_data aux_data; + hw::wallet_shim wallet_shim; + setup_shim(&wallet_shim, this); + aux_data.tx_recipients = dsts_info; + dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); + tx_device_aux = aux_data.tx_device_aux; + + MDEBUG("Signed tx data from hw: " << exported_txs.ptx.size() << " transactions"); + for (auto &c_ptx: exported_txs.ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(c_ptx.tx)); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { + auto & hwdev = get_account().get_device(); + if (!hwdev.has_ki_cold_sync()){ + throw std::invalid_argument("Device does not support cold ki sync protocol"); + } + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + std::vector<std::pair<crypto::key_image, crypto::signature>> ski; + hw::wallet_shim wallet_shim; + setup_shim(&wallet_shim, this); + + dev_cold->ki_sync(&wallet_shim, m_transfers, ski); + + return import_key_images(ski, spent, unspent); +} +//---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const { boost::optional<std::string> result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); @@ -8097,7 +9175,7 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); throw_on_rpc_response_error(result, "get_hard_fork_info"); - bool close_enough = height >= earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand + bool close_enough = height >= earliest_height - early_blocks; // start using the rules that many blocks beforehand if (close_enough) LOG_PRINT_L2("Using v" << (unsigned)version << " rules"); else @@ -8105,12 +9183,15 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const return close_enough; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_upper_transaction_size_limit() const +uint64_t wallet2::get_upper_transaction_weight_limit() const { - if (m_upper_transaction_size_limit > 0) - return m_upper_transaction_size_limit; + if (m_upper_transaction_weight_limit > 0) + return m_upper_transaction_weight_limit; uint64_t full_reward_zone = use_fork_rules(5, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : use_fork_rules(2, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; - return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + if (use_fork_rules(8, 10)) + return full_reward_zone / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } //---------------------------------------------------------------------------------------------------- std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const @@ -8148,12 +9229,12 @@ std::vector<uint64_t> wallet2::get_unspent_amounts_vector() const return vector; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon) +std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct) { cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - if (trusted_daemon) + if (is_trusted_daemon()) req_t.amounts = get_unspent_amounts_vector(); req_t.min_count = count; req_t.max_count = 0; @@ -8214,30 +9295,28 @@ const wallet2::transfer_details &wallet2::get_transfer_details(size_t idx) const return m_transfers[idx]; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_unmixable_outputs() { - // request all outputs with less than 3 instances - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, false, true, false, trusted_daemon); + // request all outputs with less instances than the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), false, true, false); } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_mixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_mixable_outputs() { - // request all outputs with at least 3 instances, so we can use mixin 2 with - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, true, true, true, trusted_daemon); + // request all outputs with at least as many instances as the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), true, true, true); } //---------------------------------------------------------------------------------------------------- -std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions() { // From hard fork 1, we don't consider small amounts to be dust anymore const bool hf1_rules = use_fork_rules(2, 10); // first hard fork has version 2 tx_dust_policy dust_policy(hf1_rules ? 0 : ::config::DEFAULT_DUST_THRESHOLD); - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); // may throw - std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); size_t num_dust_outputs = unmixable_outputs.size(); if (num_dust_outputs == 0) @@ -8249,13 +9328,23 @@ std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bo std::vector<size_t> unmixable_transfer_outputs, unmixable_dust_outputs; for (auto n: unmixable_outputs) { - if (m_transfers[n].amount() < fee_per_kb) + if (m_transfers[n].amount() < base_fee) unmixable_dust_outputs.push_back(n); else unmixable_transfer_outputs.push_back(n); } - return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>(), trusted_daemon); + return create_transactions_from(m_account_public_address, false, 1, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>()); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::discard_unmixable_outputs() +{ + // may throw + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); + for (size_t idx : unmixable_outputs) + { + m_transfers[idx].m_spent = true; + } } bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const @@ -8271,6 +9360,54 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s return true; } //---------------------------------------------------------------------------------------------------- +void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys) +{ + // fetch tx from daemon and check if secret keys agree with corresponding public keys + COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = false; + COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); + bool r; + { + const boost::lock_guard<boost::mutex> lock{m_daemon_rpc_mutex}; + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + } + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, + "daemon returned wrong response for gettransactions, wrong txs count = " + + std::to_string(res.txs.size()) + ", expected 1"); + cryptonote::blobdata bd; + THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; + crypto::hash tx_hash, tx_prefix_hash; + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + std::vector<tx_extra_field> tx_extra_fields; + THROW_WALLET_EXCEPTION_IF(!parse_tx_extra(tx.extra, tx_extra_fields), error::wallet_internal_error, "Transaction extra has unsupported format"); + tx_extra_pub_key pub_key_field; + bool found = false; + size_t index = 0; + while (find_tx_extra_field_by_type(tx_extra_fields, pub_key_field, index++)) + { + crypto::public_key calculated_pub_key; + crypto::secret_key_to_public_key(tx_key, calculated_pub_key); + if (calculated_pub_key == pub_key_field.pub_key) + { + found = true; + break; + } + } + THROW_WALLET_EXCEPTION_IF(!found, error::wallet_internal_error, "Given tx secret key doesn't agree with the tx public key in the blockchain"); + tx_extra_additional_pub_keys additional_tx_pub_keys; + find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys); + THROW_WALLET_EXCEPTION_IF(additional_tx_keys.size() != additional_tx_pub_keys.data.size(), error::wallet_internal_error, "The number of additional tx secret keys doesn't agree with the number of additional tx public keys in the blockchain" ); + m_tx_keys.insert(std::make_pair(txid, tx_key)); + m_additional_tx_keys.insert(std::make_pair(txid, additional_tx_keys)); +} +//---------------------------------------------------------------------------------------------------- std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string &message) { THROW_WALLET_EXCEPTION_IF(m_watch_only, error::wallet_internal_error, @@ -8430,8 +9567,9 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes } std::vector<std::vector<crypto::signature>> signatures = { std::vector<crypto::signature>(1) }; const size_t sig_len = tools::base58::encode(std::string((const char *)&signatures[0][0], sizeof(crypto::signature))).size(); - THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * sig_len, - error::wallet_internal_error, "incorrect signature size"); + if( sig_str.size() != header_len + num_sigs * sig_len ) { + return false; + } // decode base58 signatures.clear(); @@ -8583,6 +9721,8 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1)); const rct::key C = tx.rct_signatures.outPk[n].mask; rct::key Ctmp; + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.mask.bytes) != 0, error::wallet_internal_error, "Bad ECDH input mask"); + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.amount.bytes) != 0, error::wallet_internal_error, "Bad ECDH input amount"); rct::addKeys2(Ctmp, ecdh_info.mask, ecdh_info.amount, rct::H); if (rct::equalKeys(C, Ctmp)) amount = rct::h2d(ecdh_info.amount); @@ -8870,6 +10010,7 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, if (account_minreserve) { + THROW_WALLET_EXCEPTION_IF(account_minreserve->second == 0, error::wallet_internal_error, "Proved amount must be greater than 0"); // minimize the number of outputs included in the proof, by only picking the N largest outputs that can cover the requested min reserve amount std::sort(selected_transfers.begin(), selected_transfers.end(), [&](const size_t a, const size_t b) { return m_transfers[a].amount() > m_transfers[b].amount(); }); @@ -9131,45 +10272,29 @@ uint64_t wallet2::get_daemon_blockchain_height(string &err) const uint64_t wallet2::get_daemon_blockchain_target_height(string &err) { - cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); - m_daemon_rpc_mutex.lock(); - bool ok = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client); - m_daemon_rpc_mutex.unlock(); - if (ok) - { - if (resp_t.status == CORE_RPC_STATUS_BUSY) - { - err = "daemon is busy. Please try again later."; - } - else if (resp_t.status != CORE_RPC_STATUS_OK) - { - err = resp_t.status; - } - else // success, cleaning up error message - { - err = ""; - } - } - else + err = ""; + uint64_t target_height = 0; + const auto result = m_node_rpc_proxy.get_target_height(target_height); + if (result && *result != CORE_RPC_STATUS_OK) { - err = "possibly lost connection to daemon"; + err= *result; + return 0; } - return resp_t.target_height; + return target_height; } uint64_t wallet2::get_approximate_blockchain_height() const { // time of v2 fork - const time_t fork_time = m_nettype == TESTNET ? 1448285909 : m_nettype == STAGENET ? (time_t)-1/*TODO*/ : 1458748658; + const time_t fork_time = m_nettype == TESTNET ? 1448285909 : m_nettype == STAGENET ? 1520937818 : 1458748658; // v2 fork block - const uint64_t fork_block = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? (uint64_t)-1/*TODO*/ : 1009827; + const uint64_t fork_block = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? 32000 : 1009827; // avg seconds per block const int seconds_per_block = DIFFICULTY_TARGET_V2; // Calculated blockchain height uint64_t approx_blockchain_height = fork_block + (time(NULL) - fork_time)/seconds_per_block; // testnet got some huge rollbacks, so the estimation is way off - static const uint64_t approximate_testnet_rolled_back_blocks = 148540; + static const uint64_t approximate_testnet_rolled_back_blocks = 303967; if (m_nettype == TESTNET && approx_blockchain_height > approximate_testnet_rolled_back_blocks) approx_blockchain_height -= approximate_testnet_rolled_back_blocks; LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height); @@ -9189,6 +10314,19 @@ std::string wallet2::get_tx_note(const crypto::hash &txid) const return i->second; } +void wallet2::set_tx_device_aux(const crypto::hash &txid, const std::string &aux) +{ + m_tx_device[txid] = aux; +} + +std::string wallet2::get_tx_device_aux(const crypto::hash &txid) const +{ + std::unordered_map<crypto::hash, std::string>::const_iterator i = m_tx_device.find(txid); + if (i == m_tx_device.end()) + return std::string(); + return i->second; +} + void wallet2::set_attribute(const std::string &key, const std::string &value) { m_attributes[key] = value; @@ -9536,7 +10674,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::wallet_internal_error, "Signature check failed: input " + boost::lexical_cast<std::string>(n) + "/" + error::signature_check_failed, boost::lexical_cast<std::string>(n) + "/" + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); @@ -9573,7 +10711,18 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag std::unordered_set<crypto::hash> spent_txids; // For each spent key image, search for a tx in m_transfers that uses it as input. std::vector<size_t> swept_transfers; // If such a spending tx wasn't found in m_transfers, this means the spending tx // was created by sweep_all, so we can't know the spent height and other detailed info. - for(size_t i = 0; i < m_transfers.size(); ++i) + std::unordered_map<crypto::key_image, crypto::hash> spent_key_images; + + for (const transfer_details &td: m_transfers) + { + for (const cryptonote::txin_v& in : td.m_tx.vin) + { + if (in.type() == typeid(cryptonote::txin_to_key)) + spent_key_images.insert(std::make_pair(boost::get<cryptonote::txin_to_key>(in).k_image, td.m_txid)); + } + } + + for(size_t i = 0; i < signed_key_images.size(); ++i) { transfer_details &td = m_transfers[i]; uint64_t amount = td.amount(); @@ -9586,28 +10735,11 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag if (i < daemon_resp.spent_status.size() && daemon_resp.spent_status[i] == COMMAND_RPC_IS_KEY_IMAGE_SPENT::SPENT_IN_BLOCKCHAIN) { - bool is_spent_tx_found = false; - for (auto it = m_transfers.rbegin(); &(*it) != &td; ++it) - { - bool is_spent_tx = false; - for(const cryptonote::txin_v& in : it->m_tx.vin) - { - if(in.type() == typeid(cryptonote::txin_to_key) && td.m_key_image == boost::get<cryptonote::txin_to_key>(in).k_image) - { - is_spent_tx = true; - break; - } - } - if (is_spent_tx) - { - is_spent_tx_found = true; - spent_txids.insert(it->m_txid); - break; - } - } - - if (!is_spent_tx_found) + const std::unordered_map<crypto::key_image, crypto::hash>::const_iterator skii = spent_key_images.find(td.m_key_image); + if (skii == spent_key_images.end()) swept_transfers.push_back(i); + else + spent_txids.insert(skii->second); } } MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); @@ -9730,17 +10862,39 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag { const transfer_details& td = m_transfers[n]; confirmed_transfer_details pd; - pd.m_change = (uint64_t)-1; // cahnge is unknown + pd.m_change = (uint64_t)-1; // change is unknown pd.m_amount_in = pd.m_amount_out = td.amount(); // fee is unknown - std::string err; - pd.m_block_height = get_daemon_blockchain_height(err); // spent block height is unknown, so hypothetically set to the highest - crypto::hash spent_txid = crypto::rand<crypto::hash>(); // spent txid is unknown, so hypothetically set to random + pd.m_block_height = 0; // spent block height is unknown + const crypto::hash &spent_txid = crypto::null_hash; // spent txid is unknown m_confirmed_txs.insert(std::make_pair(spent_txid, pd)); } } return m_transfers[signed_key_images.size() - 1].m_block_height; } + +bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) +{ + if (key_images.size() > m_transfers.size()) + { + LOG_PRINT_L1("More key images returned that we know outputs for"); + return false; + } + for (size_t i = 0; i < key_images.size(); ++i) + { + transfer_details &td = m_transfers[i]; + if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[i]) + LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); + td.m_key_image = key_images[i]; + m_key_images[m_transfers[i].m_key_image] = i; + td.m_key_image_known = true; + td.m_key_image_partial = false; + m_pub_keys[m_transfers[i].get_public_key()] = i; + } + + return true; +} + wallet2::payment_container wallet2::export_payments() const { payment_container payments; @@ -9796,7 +10950,7 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect generate_genesis(genesis); crypto::hash genesis_hash = get_block_hash(genesis); check_genesis(genesis_hash); - m_local_bc_height = m_blockchain.size(); + m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const @@ -9814,6 +10968,23 @@ std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const return outs; } //---------------------------------------------------------------------------------------------------- +std::string wallet2::export_outputs_to_str() const +{ + std::vector<tools::wallet2::transfer_details> outs = export_outputs(); + + std::stringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + ar << outs; + + std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); + const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + std::string header; + header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); + header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); + std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); + return magic + ciphertext; +} +//---------------------------------------------------------------------------------------------------- size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs) { m_transfers.clear(); @@ -9829,23 +11000,86 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout[td.m_internal_output_index].target.type() != typeid(cryptonote::txout_to_key), + error::wallet_internal_error, "Unsupported output type"); const crypto::public_key& out_key = boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key; bool r = cryptonote::generate_key_image_helper(m_account.get_keys(), m_subaddresses, out_key, tx_pub_key, additional_tx_pub_keys, td.m_internal_output_index, in_ephemeral, td.m_key_image, m_account.get_device()); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; td.m_key_image_partial = false; - THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key, + THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i)); m_key_images[td.m_key_image] = m_transfers.size(); m_pub_keys[td.get_public_key()] = m_transfers.size(); - m_transfers.push_back(td); + m_transfers.push_back(std::move(td)); } return m_transfers.size(); } //---------------------------------------------------------------------------------------------------- +size_t wallet2::import_outputs_from_str(const std::string &outputs_st) +{ + std::string data = outputs_st; + const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); + if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Bad magic from outputs")); + } + + try + { + data = decrypt_with_view_secret_key(std::string(data, magiclen)); + } + catch (const std::exception &e) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt outputs: ") + e.what()); + } + + const size_t headerlen = 2 * sizeof(crypto::public_key); + if (data.size() < headerlen) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Bad data size for outputs")); + } + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Outputs from are for a different account")); + } + + size_t imported_outputs = 0; + try + { + std::string body(data, headerlen); + std::stringstream iss; + iss << body; + std::vector<tools::wallet2::transfer_details> outputs; + try + { + boost::archive::portable_binary_iarchive ar(iss); + ar >> outputs; + } + catch (...) + { + iss.str(""); + iss << body; + boost::archive::binary_iarchive ar(iss); + ar >> outputs; + } + + imported_outputs = import_outputs(outputs); + } + catch (const std::exception &e) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to import outputs") + e.what()); + } + + return imported_outputs; +} +//---------------------------------------------------------------------------------------------------- crypto::public_key wallet2::get_multisig_signer_public_key(const crypto::secret_key &spend_skey) const { crypto::public_key pkey; @@ -10118,23 +11352,29 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) m_multisig_rescan_info = &info; try { - refresh(); + + refresh(false); + } + catch (...) + { + m_multisig_rescan_info = NULL; + m_multisig_rescan_k = NULL; + throw; } - catch (...) {} m_multisig_rescan_info = NULL; m_multisig_rescan_k = NULL; return n_outputs; } //---------------------------------------------------------------------------------------------------- -std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated) const +std::string wallet2::encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated) const { crypto::chacha_key key; - crypto::generate_chacha_key(&skey, sizeof(skey), key); + crypto::generate_chacha_key(&skey, sizeof(skey), key, m_kdf_rounds); std::string ciphertext; crypto::chacha_iv iv = crypto::rand<crypto::chacha_iv>(); - ciphertext.resize(plaintext.size() + sizeof(iv) + (authenticated ? sizeof(crypto::signature) : 0)); - crypto::chacha20(plaintext.data(), plaintext.size(), key, iv, &ciphertext[sizeof(iv)]); + ciphertext.resize(len + sizeof(iv) + (authenticated ? sizeof(crypto::signature) : 0)); + crypto::chacha20(plaintext, len, key, iv, &ciphertext[sizeof(iv)]); memcpy(&ciphertext[0], &iv, sizeof(iv)); if (authenticated) { @@ -10148,22 +11388,36 @@ std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_ return ciphertext; } //---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const epee::span<char> &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const epee::wipeable_string &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- std::string wallet2::encrypt_with_view_secret_key(const std::string &plaintext, bool authenticated) const { return encrypt(plaintext, get_account().get_keys().m_view_secret_key, authenticated); } //---------------------------------------------------------------------------------------------------- -std::string wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const +template<typename T> +T wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const { const size_t prefix_size = sizeof(chacha_iv) + (authenticated ? sizeof(crypto::signature) : 0); THROW_WALLET_EXCEPTION_IF(ciphertext.size() < prefix_size, error::wallet_internal_error, "Unexpected ciphertext size"); crypto::chacha_key key; - crypto::generate_chacha_key(&skey, sizeof(skey), key); + crypto::generate_chacha_key(&skey, sizeof(skey), key, m_kdf_rounds); const crypto::chacha_iv &iv = *(const crypto::chacha_iv*)&ciphertext[0]; - std::string plaintext; - plaintext.resize(ciphertext.size() - prefix_size); if (authenticated) { crypto::hash hash; @@ -10174,10 +11428,14 @@ std::string wallet2::decrypt(const std::string &ciphertext, const crypto::secret THROW_WALLET_EXCEPTION_IF(!crypto::check_signature(hash, pkey, signature), error::wallet_internal_error, "Failed to authenticate ciphertext"); } - crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - prefix_size, key, iv, &plaintext[0]); - return plaintext; + std::unique_ptr<char[]> buffer{new char[ciphertext.size() - prefix_size]}; + auto wiper = epee::misc_utils::create_scope_leave_handler([&]() { memwipe(buffer.get(), ciphertext.size() - prefix_size); }); + crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - prefix_size, key, iv, buffer.get()); + return T(buffer.get(), ciphertext.size() - prefix_size); } //---------------------------------------------------------------------------------------------------- +template epee::wipeable_string wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const; +//---------------------------------------------------------------------------------------------------- std::string wallet2::decrypt_with_view_secret_key(const std::string &ciphertext, bool authenticated) const { return decrypt(ciphertext, get_account().get_keys().m_view_secret_key, authenticated); @@ -10435,50 +11693,46 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); - m_daemon_rpc_mutex.lock(); - r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); - THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - uint64_t full_reward_zone = resp_t.block_size_limit / 2; + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); + throw_on_rpc_response_error(result, "get_info"); + uint64_t full_reward_zone = block_weight_limit / 2; + THROW_WALLET_EXCEPTION_IF(full_reward_zone == 0, error::wallet_internal_error, "Invalid block weight limit from daemon"); std::vector<std::pair<uint64_t, uint64_t>> blocks; for (const auto &fee_level: fee_levels) { const double our_fee_byte_min = fee_level.first; const double our_fee_byte_max = fee_level.second; - uint64_t priority_size_min = 0, priority_size_max = 0; + uint64_t priority_weight_min = 0, priority_weight_max = 0; for (const auto &i: res.backlog) { - if (i.blob_size == 0) + if (i.weight == 0) { - MWARNING("Got 0 sized blob from txpool, ignored"); + MWARNING("Got 0 weight tx from txpool, ignored"); continue; } - double this_fee_byte = i.fee / (double)i.blob_size; + double this_fee_byte = i.fee / (double)i.weight; if (this_fee_byte >= our_fee_byte_min) - priority_size_min += i.blob_size; + priority_weight_min += i.weight; if (this_fee_byte >= our_fee_byte_max) - priority_size_max += i.blob_size; + priority_weight_max += i.weight; } - uint64_t nblocks_min = priority_size_min / full_reward_zone; - uint64_t nblocks_max = priority_size_max / full_reward_zone; - MDEBUG("estimate_backlog: priority_size " << priority_size_min << " - " << priority_size_max << " for " + uint64_t nblocks_min = priority_weight_min / full_reward_zone; + uint64_t nblocks_max = priority_weight_max / full_reward_zone; + MDEBUG("estimate_backlog: priority_weight " << priority_weight_min << " - " << priority_weight_max << " for " << our_fee_byte_min << " - " << our_fee_byte_max << " piconero byte fee, " - << nblocks_min << " - " << nblocks_max << " blocks at block size " << full_reward_zone); + << nblocks_min << " - " << nblocks_max << " blocks at block weight " << full_reward_zone); blocks.push_back(std::make_pair(nblocks_min, nblocks_max)); } return blocks; } //---------------------------------------------------------------------------------------------------- -std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees) +std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees) { - THROW_WALLET_EXCEPTION_IF(min_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); - THROW_WALLET_EXCEPTION_IF(max_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(min_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(max_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); for (uint64_t fee: fees) { THROW_WALLET_EXCEPTION_IF(fee == 0, error::wallet_internal_error, "Invalid 0 fee"); @@ -10486,7 +11740,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t mi std::vector<std::pair<double, double>> fee_levels; for (uint64_t fee: fees) { - double our_fee_byte_min = fee / (double)min_blob_size, our_fee_byte_max = fee / (double)max_blob_size; + double our_fee_byte_min = fee / (double)min_tx_weight, our_fee_byte_max = fee / (double)max_tx_weight; fee_levels.emplace_back(our_fee_byte_min, our_fee_byte_max); } return estimate_backlog(fee_levels); @@ -10545,17 +11799,6 @@ uint64_t wallet2::get_segregation_fork_height() const } //---------------------------------------------------------------------------------------------------- void wallet2::generate_genesis(cryptonote::block& b) const { - if (m_nettype == TESTNET) - { - cryptonote::generate_genesis_block(b, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else if (m_nettype == STAGENET) - { - cryptonote::generate_genesis_block(b, config::stagenet::GENESIS_TX, config::stagenet::GENESIS_NONCE); - } - else - { - cryptonote::generate_genesis_block(b, config::GENESIS_TX, config::GENESIS_NONCE); - } + cryptonote::generate_genesis_block(b, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); } } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 40f6e08d9..7d78a3fee 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -54,6 +54,7 @@ #include "ringct/rctTypes.h" #include "ringct/rctOps.h" #include "checkpoints/checkpoints.h" +#include "serialization/pair.h" #include "wallet_errors.h" #include "common/password.h" @@ -67,6 +68,20 @@ class Serialization_portability_wallet_Test; namespace tools { class ringdb; + class wallet2; + class Notify; + + class wallet_keys_unlocker + { + public: + wallet_keys_unlocker(wallet2 &w, const boost::optional<tools::password_container> &password); + wallet_keys_unlocker(wallet2 &w, bool locked, const epee::wipeable_string &password); + ~wallet_keys_unlocker(); + private: + wallet2 &w; + bool locked; + crypto::chacha_key key; + }; class i_wallet2_callback { @@ -77,6 +92,7 @@ namespace tools virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx) {} + virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason) { return boost::none; } // Light wallet callbacks virtual void on_lw_new_block(uint64_t height) {} virtual void on_lw_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} @@ -133,9 +149,11 @@ namespace tools std::deque<crypto::hash> m_blockchain; }; + class wallet_keys_unlocker; class wallet2 { friend class ::Serialization_portability_wallet_Test; + friend class wallet_keys_unlocker; public: static constexpr const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); @@ -146,28 +164,36 @@ namespace tools RefreshDefault = RefreshOptimizeCoinbase, }; + enum AskPasswordType { + AskPasswordNever = 0, + AskPasswordOnAction = 1, + AskPasswordToDecrypt = 2, + }; + static const char* tr(const char* str); static bool has_testnet_option(const boost::program_options::variables_map& vm); static bool has_stagenet_option(const boost::program_options::variables_map& vm); + static std::string device_name_option(const boost::program_options::variables_map& vm); static void init_options(boost::program_options::options_description& desc_params); //! Uses stdin and stdout. Returns a wallet2 if no errors. - static std::unique_ptr<wallet2> make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::pair<std::unique_ptr<wallet2>, password_container> make_from_json(const boost::program_options::variables_map& vm, bool unattended, const std::string& json_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Uses stdin and stdout. Returns a wallet2 and password for `wallet_file` if no errors. static std::pair<std::unique_ptr<wallet2>, password_container> - make_from_file(const boost::program_options::variables_map& vm, const std::string& wallet_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + make_from_file(const boost::program_options::variables_map& vm, bool unattended, const std::string& wallet_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Uses stdin and stdout. Returns a wallet2 and password for wallet with no file if no errors. - static std::pair<std::unique_ptr<wallet2>, password_container> make_new(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::pair<std::unique_ptr<wallet2>, password_container> make_new(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Just parses variables. - static std::unique_ptr<wallet2> make_dummy(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::unique_ptr<wallet2> make_dummy(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); - static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev); + static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds); + static bool query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds = 1); - wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, bool restricted = false); + wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false); ~wallet2(); struct multisig_info @@ -260,12 +286,12 @@ namespace tools uint64_t m_block_height; uint64_t m_unlock_time; uint64_t m_timestamp; + bool m_coinbase; cryptonote::subaddress_index m_subaddr_index; }; struct address_tx : payment_details { - bool m_coinbase; bool m_mempool; bool m_incoming; }; @@ -320,6 +346,7 @@ namespace tools std::vector<uint8_t> extra; uint64_t unlock_time; bool use_rct; + bool use_bulletproofs; std::vector<cryptonote::tx_destination_entry> dests; // original setup, does not include change uint32_t subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> subaddr_indices; // set of address indices used as inputs in this transfer @@ -332,6 +359,7 @@ namespace tools FIELD(extra) FIELD(unlock_time) FIELD(use_rct) + FIELD(use_bulletproofs) FIELD(dests) FIELD(subaddr_account) FIELD(subaddr_indices) @@ -452,6 +480,29 @@ namespace tools typedef std::tuple<uint64_t, crypto::public_key, rct::key> get_outs_entry; + struct parsed_block + { + crypto::hash hash; + cryptonote::block block; + std::vector<cryptonote::transaction> txes; + cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices o_indices; + bool error; + }; + + struct is_out_data + { + crypto::public_key pkey; + crypto::key_derivation derivation; + std::vector<boost::optional<cryptonote::subaddress_receive_info>> received; + }; + + struct tx_cache_data + { + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + std::vector<is_out_data> primary; + std::vector<is_out_data> additional; + }; + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -460,7 +511,7 @@ namespace tools * \param create_address_file Whether to create an address file */ void generate(const std::string& wallet_, const epee::wipeable_string& password, - const std::string& multisig_data, bool create_address_file = false); + const epee::wipeable_string& multisig_data, bool create_address_file = false); /*! * \brief Generates a wallet or restores one. @@ -503,8 +554,9 @@ namespace tools * \param wallet_ Name of wallet file * \param password Password of wallet file * \param device_name name of HW to use + * \param create_address_file Whether to create an address file */ - void restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name); + void restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name, bool create_address_file = false); /*! * \brief Creates a multisig wallet @@ -523,6 +575,14 @@ namespace tools const std::vector<crypto::secret_key> &view_keys, const std::vector<crypto::public_key> &spend_keys, uint32_t threshold); + std::string exchange_multisig_keys(const epee::wipeable_string &password, + const std::vector<std::string> &info); + /*! + * \brief Any but first round of keys exchange + */ + std::string exchange_multisig_keys(const epee::wipeable_string &password, + std::unordered_set<crypto::public_key> pkeys, + std::vector<crypto::public_key> signers); /*! * \brief Finalizes creation of a multisig wallet */ @@ -574,35 +634,38 @@ namespace tools /*! * \brief verifies given password is correct for default wallet keys file */ - bool verify_password(const epee::wipeable_string& password) const; + bool verify_password(const epee::wipeable_string& password); cryptonote::account_base& get_account(){return m_account;} const cryptonote::account_base& get_account()const{return m_account;} + void encrypt_keys(const crypto::chacha_key &key); + void encrypt_keys(const epee::wipeable_string &password); + void decrypt_keys(const crypto::chacha_key &key); + void decrypt_keys(const epee::wipeable_string &password); + void set_refresh_from_block_height(uint64_t height) {m_refresh_from_block_height = height;} uint64_t get_refresh_from_block_height() const {return m_refresh_from_block_height;} void explicit_refresh_from_block_height(bool expl) {m_explicit_refresh_from_block_height = expl;} bool explicit_refresh_from_block_height() const {return m_explicit_refresh_from_block_height;} - // upper_transaction_size_limit as defined below is set to - // approximately 125% of the fixed minimum allowable penalty - // free block size. TODO: fix this so that it actually takes - // into account the current median block size rather than - // the minimum block size. bool deinit(); bool init(std::string daemon_address = "http://localhost:8080", - boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_size_limit = 0, bool ssl = false); + boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, bool ssl = false, bool trusted_daemon = false); void stop() { m_run.store(false, std::memory_order_relaxed); } i_wallet2_callback* callback() const { return m_callback; } void callback(i_wallet2_callback* callback) { m_callback = callback; } + bool is_trusted_daemon() const { return m_trusted_daemon; } + void set_trusted_daemon(bool trusted) { m_trusted_daemon = trusted; } + /*! * \brief Checks if deterministic wallet */ bool is_deterministic() const; - bool get_seed(std::string& electrum_words, const epee::wipeable_string &passphrase = epee::wipeable_string()) const; + bool get_seed(epee::wipeable_string& electrum_words, const epee::wipeable_string &passphrase = epee::wipeable_string()) const; /*! * \brief Checks if light wallet. A light wallet sends view key to a server where the blockchain is scanned. @@ -624,6 +687,7 @@ namespace tools // Subaddress scheme cryptonote::account_public_address get_subaddress(const cryptonote::subaddress_index& index) const; cryptonote::account_public_address get_address() const { return get_subaddress({0,0}); } + boost::optional<cryptonote::subaddress_index> get_subaddress_index(const cryptonote::account_public_address& address) const; crypto::public_key get_subaddress_spend_public_key(const cryptonote::subaddress_index& index) const; std::vector<crypto::public_key> get_subaddress_spend_public_keys(uint32_t account, uint32_t begin, uint32_t end) const; std::string get_subaddress_as_str(const cryptonote::subaddress_index& index) const; @@ -642,21 +706,23 @@ namespace tools * \brief Tells if the wallet file is deprecated. */ bool is_deprecated() const; - void refresh(); - void refresh(uint64_t start_height, uint64_t & blocks_fetched); - void refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money); - bool refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok); + void refresh(bool trusted_daemon); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money); + bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok); void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; } RefreshType get_refresh_type() const { return m_refresh_type; } cryptonote::network_type nettype() const { return m_nettype; } - bool restricted() const { return m_restricted; } bool watch_only() const { return m_watch_only; } bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const; bool has_multisig_partial_key_images() const; - bool get_multisig_seed(std::string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const; - bool key_on_device() const { return m_key_on_device; } + bool has_unknown_key_images() const; + bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const; + bool key_on_device() const { return get_device_type() != hw::device::device_type::SOFTWARE; } + hw::device::device_type get_device_type() const { return m_key_device_type; } + bool reconnect_device(); // locked & unlocked balance of given or current subaddress account uint64_t balance(uint32_t subaddr_index_major) const; @@ -668,22 +734,17 @@ namespace tools uint64_t balance_all() const; uint64_t unlocked_balance_all() const; template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon); - template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); - template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); void transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof); + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type); void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); bool save_tx(const std::vector<pending_tx>& ptx_vector, const std::string &filename) const; + std::string dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) const; std::string save_multisig_tx(multisig_tx_set txs); bool save_multisig_tx(const multisig_tx_set &txs, const std::string &filename); std::string save_multisig_tx(const std::vector<pending_tx>& ptx_vector); @@ -693,20 +754,27 @@ namespace tools bool sign_tx(const std::string &unsigned_filename, const std::string &signed_filename, std::vector<wallet2::pending_tx> &ptx, std::function<bool(const unsigned_tx_set&)> accept_func = NULL, bool export_raw = false); // sign unsigned tx. Takes unsigned_tx_set as argument. Used by GUI bool sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_filename, std::vector<wallet2::pending_tx> &ptx, bool export_raw = false); + bool sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &ptx, signed_tx_set &signed_txs); + std::string sign_tx_dump_to_str(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &ptx, signed_tx_set &signed_txes); // load unsigned_tx_set from file. bool load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx_set &exported_txs) const; + bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const; bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL); - std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); // pass subaddr_indices by value on purpose - std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); + bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func); + std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); // pass subaddr_indices by value on purpose + std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); + std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); + void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); + uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); bool sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto::hash> &txids); bool sign_multisig_tx_to_file(multisig_tx_set &exported_txs, const std::string &filename, std::vector<crypto::hash> &txids); - std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon); + std::vector<pending_tx> create_unmixable_sweep_transactions(); + void discard_unmixable_outputs(); bool check_connection(uint32_t *version = NULL, uint32_t timeout = 200000); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; @@ -716,11 +784,14 @@ namespace tools void get_unconfirmed_payments_out(std::list<std::pair<crypto::hash,wallet2::unconfirmed_transfer_details>>& unconfirmed_payments, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; void get_unconfirmed_payments(std::list<std::pair<crypto::hash,wallet2::pool_payment_details>>& unconfirmed_payments, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; - uint64_t get_blockchain_current_height() const { return m_local_bc_height; } + uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); void rescan_blockchain(bool refresh = true); bool is_transfer_unlocked(const transfer_details& td) const; bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; + + uint64_t get_last_block_reward() const { return m_last_block_reward; } + template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) { @@ -822,6 +893,12 @@ namespace tools if(ver < 24) return; a & m_ring_history_saved; + if(ver < 25) + return; + a & m_last_block_reward; + if(ver < 26) + return; + a & m_tx_device; } /*! @@ -855,8 +932,8 @@ namespace tools void auto_refresh(bool r) { m_auto_refresh = r; } bool confirm_missing_payment_id() const { return m_confirm_missing_payment_id; } void confirm_missing_payment_id(bool always) { m_confirm_missing_payment_id = always; } - bool ask_password() const { return m_ask_password; } - void ask_password(bool always) { m_ask_password = always; } + AskPasswordType ask_password() const { return m_ask_password; } + void ask_password(AskPasswordType ask) { m_ask_password = ask; } void set_min_output_count(uint32_t count) { m_min_output_count = count; } uint32_t get_min_output_count() const { return m_min_output_count; } void set_min_output_value(uint64_t value) { m_min_output_value = value; } @@ -877,10 +954,15 @@ namespace tools void key_reuse_mitigation2(bool value) { m_key_reuse_mitigation2 = value; } uint64_t segregation_height() const { return m_segregation_height; } void segregation_height(uint64_t height) { m_segregation_height = height; } + bool ignore_fractional_outputs() const { return m_ignore_fractional_outputs; } + void ignore_fractional_outputs(bool value) { m_ignore_fractional_outputs = value; } bool confirm_non_default_ring_size() const { return m_confirm_non_default_ring_size; } void confirm_non_default_ring_size(bool always) { m_confirm_non_default_ring_size = always; } + const std::string & device_name() const { return m_device_name; } + void device_name(const std::string & device_name) { m_device_name = device_name; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; + void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); void check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); std::string get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message); @@ -934,10 +1016,10 @@ namespace tools */ uint64_t get_approximate_blockchain_height() const; uint64_t estimate_blockchain_height(); - std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon); + std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct); std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const; - std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon); - std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon); + std::vector<size_t> select_available_unmixable_outputs(); + std::vector<size_t> select_available_mixable_outputs(); size_t pop_best_value_from(const transfer_container &transfers, std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; size_t pop_best_value(std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; @@ -945,6 +1027,9 @@ namespace tools void set_tx_note(const crypto::hash &txid, const std::string ¬e); std::string get_tx_note(const crypto::hash &txid) const; + void set_tx_device_aux(const crypto::hash &txid, const std::string &aux); + std::string get_tx_device_aux(const crypto::hash &txid) const; + void set_description(const std::string &description); std::string get_description() const; @@ -987,7 +1072,9 @@ namespace tools // Import/Export wallet data std::vector<tools::wallet2::transfer_details> export_outputs() const; + std::string export_outputs_to_str() const; size_t import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs); + size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); void import_payments_out(const std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> &confirmed_payments); @@ -997,13 +1084,18 @@ namespace tools std::vector<std::pair<crypto::key_image, crypto::signature>> export_key_images() const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); + bool import_key_images(std::vector<crypto::key_image> key_images); + crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; void update_pool_state(bool refreshed = false); void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes); + std::string encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated = true) const; + std::string encrypt(const epee::span<char> &span, const crypto::secret_key &skey, bool authenticated = true) const; std::string encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated = true) const; + std::string encrypt(const epee::wipeable_string &plaintext, const crypto::secret_key &skey, bool authenticated = true) const; std::string encrypt_with_view_secret_key(const std::string &plaintext, bool authenticated = true) const; - std::string decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated = true) const; + template<typename T=std::string> T decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated = true) const; std::string decrypt_with_view_secret_key(const std::string &ciphertext, bool authenticated = true) const; std::string make_uri(const std::string &address, const std::string &payment_id, uint64_t amount, const std::string &tx_description, const std::string &recipient_name, std::string &error) const; @@ -1014,13 +1106,18 @@ namespace tools bool is_synced() const; std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(const std::vector<std::pair<double, double>> &fee_levels); - std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees); + std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees); uint64_t get_fee_multiplier(uint32_t priority, int fee_algorithm = -1) const; - uint64_t get_per_kb_fee() const; + uint64_t get_base_fee() const; + uint64_t get_fee_quantization_mask() const; + uint64_t get_min_ring_size() const; + uint64_t get_max_ring_size() const; uint64_t adjust_mixin(uint64_t mixin) const; uint32_t adjust_priority(uint32_t priority); + bool is_unattended() const { return m_unattended; } + // Light wallet specific functions // fetch unspent outs from lw node and store in m_transfers void light_wallet_get_unspent_outs(); @@ -1089,10 +1186,18 @@ namespace tools bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); bool find_and_save_rings(bool force = true); - bool blackball_output(const crypto::public_key &output); - bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false); - bool unblackball_output(const crypto::public_key &output); - bool is_output_blackballed(const crypto::public_key &output) const; + bool blackball_output(const std::pair<uint64_t, uint64_t> &output); + bool set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add = false); + bool unblackball_output(const std::pair<uint64_t, uint64_t> &output); + bool is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const; + + bool lock_keys_file(); + bool unlock_keys_file(); + bool is_keys_file_locked() const; + + void change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password); + + void set_tx_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_tx_notify = notify; } private: /*! @@ -1109,18 +1214,18 @@ namespace tools * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); - void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen); - void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices); + void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data); + void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset); void detach_blockchain(uint64_t height); - void get_short_chain_history(std::list<crypto::hash>& ids) const; + void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; bool clear(); - void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); - void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes); - void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history); - void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, bool &error); - void process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, const std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t& blocks_added); - uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const; + void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); + void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); + void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); + void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error); + void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added); + uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); void process_outgoing(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received, uint32_t subaddr_account, const std::set<uint32_t>& subaddr_indices); @@ -1128,22 +1233,24 @@ namespace tools void generate_genesis(cryptonote::block& b) const; void check_genesis(const crypto::hash& genesis_hash) const; //throws bool generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const; + void generate_chacha_key_from_password(const epee::wipeable_string &pass, crypto::chacha_key &key) const; crypto::hash get_payment_id(const pending_tx &ptx) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, tx_scan_info_t &tx_scan_info) const; + void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const; + void check_acc_out_precomp_once(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; - uint64_t get_upper_transaction_size_limit() const; + uint64_t get_upper_transaction_weight_limit() const; std::vector<uint64_t> get_unspent_amounts_vector() const; - uint64_t get_dynamic_per_kb_fee_estimate() const; + uint64_t get_dynamic_base_fee_estimate() const; float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const; std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices) const; void set_spent(size_t idx, uint64_t height); void set_unspent(size_t idx); void get_outs(std::vector<std::vector<get_outs_entry>> &outs, const std::vector<size_t> &selected_transfers, size_t fake_outputs_count); bool tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; - crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; std::vector<size_t> get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const; - void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) const; + void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); void trim_hashchain(); crypto::key_image get_multisig_composite_key_image(size_t n) const; rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const crypto::public_key &ignore, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; @@ -1154,10 +1261,26 @@ namespace tools bool add_rings(const cryptonote::transaction_prefix &tx); bool remove_rings(const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); + crypto::chacha_key get_ringdb_key(); + void setup_keys(const epee::wipeable_string &password); + + void register_devices(); + hw::device& lookup_device(const std::string & device_descriptor); - bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); + bool get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); uint64_t get_segregation_fork_height() const; + void unpack_multisig_info(const std::vector<std::string>& info, + std::vector<crypto::public_key> &public_keys, + std::vector<crypto::secret_key> &secret_keys) const; + bool unpack_extra_multisig_info(const std::vector<std::string>& info, + std::vector<crypto::public_key> &signers, + std::unordered_set<crypto::public_key> &pkeys) const; + + void cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const; + + void setup_new_blockchain(); + void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file); cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; @@ -1166,7 +1289,6 @@ namespace tools std::string m_keys_file; epee::net_utils::http::http_simple_client m_http_client; hashchain m_blockchain; - std::atomic<uint64_t> m_local_bc_height; //temporary workaround std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; std::unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments; @@ -1185,7 +1307,7 @@ namespace tools std::unordered_map<std::string, std::string> m_attributes; std::vector<tools::wallet2::address_book_row> m_address_book; std::pair<std::map<std::string, std::string>, std::vector<std::string>> m_account_tags; - uint64_t m_upper_transaction_size_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value + uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; @@ -1193,16 +1315,20 @@ namespace tools boost::mutex m_daemon_rpc_mutex; + bool m_trusted_daemon; i_wallet2_callback* m_callback; - bool m_key_on_device; + hw::device::device_type m_key_device_type; cryptonote::network_type m_nettype; - bool m_restricted; + uint64_t m_kdf_rounds; std::string seed_language; /*!< Language of the mnemonics (seed). */ bool is_old_file_format; /*!< Whether the wallet file is of an old file format */ bool m_watch_only; /*!< no spend key */ bool m_multisig; /*!< if > 1 spend secret key will not match spend public key */ uint32_t m_multisig_threshold; std::vector<crypto::public_key> m_multisig_signers; + //in case of general M/N multisig wallet we should perform N - M + 1 key exchange rounds and remember how many rounds are passed. + uint32_t m_multisig_rounds_passed; + std::vector<crypto::public_key> m_multisig_derivations; bool m_always_confirm_transfers; bool m_print_ring_members; bool m_store_tx_info; /*!< request txkey to be returned in RPC, and store in the wallet cache file */ @@ -1210,13 +1336,14 @@ namespace tools uint32_t m_default_priority; RefreshType m_refresh_type; bool m_auto_refresh; + bool m_first_refresh_done; uint64_t m_refresh_from_block_height; // If m_refresh_from_block_height is explicitly set to zero we need this to differentiate it from the case that // m_refresh_from_block_height was defaulted to zero.*/ bool m_explicit_refresh_from_block_height; bool m_confirm_missing_payment_id; bool m_confirm_non_default_ring_size; - bool m_ask_password; + AskPasswordType m_ask_password; uint32_t m_min_output_count; uint64_t m_min_output_value; bool m_merge_destinations; @@ -1227,10 +1354,15 @@ namespace tools bool m_segregate_pre_fork_outputs; bool m_key_reuse_mitigation2; uint64_t m_segregation_height; + bool m_ignore_fractional_outputs; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; size_t m_subaddress_lookahead_major, m_subaddress_lookahead_minor; + std::string m_device_name; + + // Aux transaction data from device + std::unordered_map<crypto::hash, std::string> m_tx_device; // Light wallet bool m_light_wallet; /* sends view key to daemon for scanning */ @@ -1249,14 +1381,26 @@ namespace tools std::string m_ring_database; bool m_ring_history_saved; std::unique_ptr<ringdb> m_ringdb; + boost::optional<crypto::chacha_key> m_ringdb_key; + + uint64_t m_last_block_reward; + std::unique_ptr<tools::file_locker> m_keys_file_locker; + + crypto::chacha_key m_cache_key; + boost::optional<epee::wipeable_string> m_encrypt_keys_after_refresh; + + bool m_unattended; + bool m_devices_registered; + + std::shared_ptr<tools::Notify> m_tx_notify; }; } -BOOST_CLASS_VERSION(tools::wallet2, 24) +BOOST_CLASS_VERSION(tools::wallet2, 26) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) -BOOST_CLASS_VERSION(tools::wallet2::payment_details, 3) +BOOST_CLASS_VERSION(tools::wallet2::payment_details, 4) BOOST_CLASS_VERSION(tools::wallet2::pool_payment_details, 1) BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8) BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) @@ -1264,7 +1408,7 @@ BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 0) -BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 2) +BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1523,16 +1667,24 @@ namespace boost a & x.m_timestamp; if (ver < 2) { + x.m_coinbase = false; x.m_subaddr_index = {}; return; } a & x.m_subaddr_index; if (ver < 3) { + x.m_coinbase = false; x.m_fee = 0; return; } a & x.m_fee; + if (ver < 4) + { + x.m_coinbase = false; + return; + } + a & x.m_coinbase; } template <class Archive> @@ -1611,6 +1763,9 @@ namespace boost if (ver < 2) return; a & x.selected_transfers; + if (ver < 3) + return; + a & x.use_bulletproofs; } template <class Archive> @@ -1712,197 +1867,4 @@ namespace tools //---------------------------------------------------------------------------------------------------- } //---------------------------------------------------------------------------------------------------- - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon) - { - pending_tx ptx; - cryptonote::transaction tx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx, trusted_daemon); - } - - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, bool trusted_daemon) - { - using namespace cryptonote; - // throw if attempting a transaction with no destinations - THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - - THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); - uint64_t needed_money = fee; - - // calculate total amount being sent to all destinations - // throw if total amount overflows uint64_t - for(auto& dt: dsts) - { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); - needed_money += dt.amount; - THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); - } - - // randomly select inputs for transaction - // throw if requested send amount is greater than (unlocked) amount available to send - std::vector<size_t> selected_transfers; - uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers, trusted_daemon); - THROW_WALLET_EXCEPTION_IF(found_money < needed_money, error::not_enough_unlocked_money, found_money, needed_money - fee, fee); - - uint32_t subaddr_account = m_transfers[*selected_transfers.begin()].m_subaddr_index.major; - for (auto i = ++selected_transfers.begin(); i != selected_transfers.end(); ++i) - THROW_WALLET_EXCEPTION_IF(subaddr_account != *i, error::wallet_internal_error, "the tx uses funds from multiple accounts"); - - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - typedef cryptonote::tx_source_entry::output_entry tx_output_entry; - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - if(fake_outputs_count) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req = AUTO_VAL_INIT(req); - req.outs_count = fake_outputs_count + 1;// add one to make possible (if need) to skip real output key - for(size_t idx: selected_transfers) - { - const transfer_container::const_iterator it = m_transfers.begin() + idx; - THROW_WALLET_EXCEPTION_IF(it->m_tx.vout.size() <= it->m_internal_output_index, error::wallet_internal_error, - "m_internal_output_index = " + std::to_string(it->m_internal_output_index) + - " is greater or equal to outputs count = " + std::to_string(it->m_tx.vout.size())); - req.amounts.push_back(it->amount()); - } - - m_daemon_rpc_mutex.lock(); - bool r = epee::net_utils::invoke_http_bin("/getrandom_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); - THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != selected_transfers.size(), error::wallet_internal_error, - "daemon returned wrong response for getrandom_outs.bin, wrong amounts count = " + - std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(selected_transfers.size())); - - std::unordered_map<uint64_t, uint64_t> scanty_outs; - for(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& amount_outs: daemon_resp.outs) - { - if (amount_outs.outs.size() < fake_outputs_count) - { - scanty_outs[amount_outs.amount] = amount_outs.outs.size(); - } - } - THROW_WALLET_EXCEPTION_IF(!scanty_outs.empty(), error::not_enough_outs_to_mix, scanty_outs, fake_outputs_count); - } - - //prepare inputs - size_t i = 0; - std::vector<cryptonote::tx_source_entry> sources; - for(size_t idx: selected_transfers) - { - sources.resize(sources.size()+1); - cryptonote::tx_source_entry& src = sources.back(); - const transfer_details& td = m_transfers[idx]; - src.amount = td.amount(); - src.rct = false; - //paste mixin transaction - if(daemon_resp.outs.size()) - { - daemon_resp.outs[i].outs.sort([](const out_entry& a, const out_entry& b){return a.global_amount_index < b.global_amount_index;}); - for(out_entry& daemon_oe: daemon_resp.outs[i].outs) - { - if(td.m_global_output_index == daemon_oe.global_amount_index) - continue; - tx_output_entry oe; - oe.first = daemon_oe.global_amount_index; - oe.second.dest = rct::pk2rct(daemon_oe.out_key); - oe.second.mask = rct::identity(); - src.outputs.push_back(oe); - if(src.outputs.size() >= fake_outputs_count) - break; - } - } - - //paste real transaction to the random index - auto it_to_insert = std::find_if(src.outputs.begin(), src.outputs.end(), [&](const tx_output_entry& a) - { - return a.first >= td.m_global_output_index; - }); - //size_t real_index = src.outputs.size() ? (rand() % src.outputs.size() ):0; - tx_output_entry real_oe; - real_oe.first = td.m_global_output_index; - real_oe.second.dest = rct::pk2rct(boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key); - real_oe.second.mask = rct::identity(); - auto interted_it = src.outputs.insert(it_to_insert, real_oe); - src.real_out_tx_key = get_tx_pub_key_from_extra(td.m_tx); - src.real_output = interted_it - src.outputs.begin(); - src.real_output_in_tx_index = td.m_internal_output_index; - src.multisig_kLRki = rct::multisig_kLRki({rct::zero(), rct::zero(), rct::zero(), rct::zero()}); - detail::print_source_entry(src); - ++i; - } - - cryptonote::tx_destination_entry change_dts = AUTO_VAL_INIT(change_dts); - if (needed_money < found_money) - { - change_dts.addr = get_subaddress({subaddr_account, 0}); - change_dts.amount = found_money - needed_money; - } - - std::vector<cryptonote::tx_destination_entry> splitted_dsts, dust_dsts; - uint64_t dust = 0; - destination_split_strategy(dsts, change_dts, dust_policy.dust_threshold, splitted_dsts, dust_dsts); - for(auto& d: dust_dsts) { - THROW_WALLET_EXCEPTION_IF(dust_policy.dust_threshold < d.amount, error::wallet_internal_error, "invalid dust value: dust = " + - std::to_string(d.amount) + ", dust_threshold = " + std::to_string(dust_policy.dust_threshold)); - } - for(auto& d: dust_dsts) { - if (!dust_policy.add_to_fee) - splitted_dsts.push_back(cryptonote::tx_destination_entry(d.amount, dust_policy.addr_for_dust, d.is_subaddress)); - dust += d.amount; - } - - crypto::secret_key tx_key; - std::vector<crypto::secret_key> additional_tx_keys; - rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); - - std::string key_images; - bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool - { - CHECKED_GET_SPECIFIC_VARIANT(s_e, const txin_to_key, in, false); - key_images += boost::to_string(in.k_image) + " "; - return true; - }); - THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx); - - bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key - || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key); - - if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust; - - ptx.key_images = key_images; - ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee); - ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0); - ptx.dust_added_to_fee = dust_policy.add_to_fee; - ptx.tx = tx; - ptx.change_dts = change_dts; - ptx.selected_transfers = selected_transfers; - ptx.tx_key = tx_key; - ptx.additional_tx_keys = additional_tx_keys; - ptx.dests = dsts; - ptx.construction_data.sources = sources; - ptx.construction_data.change_dts = change_dts; - ptx.construction_data.splitted_dsts = splitted_dsts; - ptx.construction_data.selected_transfers = selected_transfers; - ptx.construction_data.extra = tx.extra; - ptx.construction_data.unlock_time = unlock_time; - ptx.construction_data.use_rct = false; - ptx.construction_data.dests = dsts; - // record which subaddress indices are being used as inputs - ptx.construction_data.subaddr_account = subaddr_account; - ptx.construction_data.subaddr_indices.clear(); - for (size_t idx: selected_transfers) - ptx.construction_data.subaddr_indices.insert(m_transfers[idx].m_subaddr_index.minor); - } - - } diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index 6311e7700..95a4e0ad6 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -101,6 +101,7 @@ namespace wallet_args const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::size_t> arg_max_log_file_size = {"max-log-file-size", "Specify maximum log file size [B]", MAX_LOG_FILE_SIZE}; + const command_line::arg_descriptor<std::size_t> arg_max_log_files = {"max-log-files", "Specify maximum number of rotated log files to be saved (no limit by setting to 0)", MAX_LOG_FILES}; const command_line::arg_descriptor<uint32_t> arg_max_concurrency = {"max-concurrency", wallet_args::tr("Max number of threads to use for a parallel job"), DEFAULT_MAX_CONCURRENCY}; const command_line::arg_descriptor<std::string> arg_log_file = {"log-file", wallet_args::tr("Specify log file"), ""}; const command_line::arg_descriptor<std::string> arg_config_file = {"config-file", wallet_args::tr("Config file"), "", true}; @@ -108,6 +109,9 @@ namespace wallet_args std::string lang = i18n_get_language(); tools::on_startup(); +#ifdef NDEBUG + tools::disable_core_dumps(); +#endif tools::set_strict_default_file_permissions(true); epee::string_tools::set_module_name_and_folder(argv[0]); @@ -119,6 +123,7 @@ namespace wallet_args command_line::add_arg(desc_params, arg_log_file); command_line::add_arg(desc_params, arg_log_level); command_line::add_arg(desc_params, arg_max_log_file_size); + command_line::add_arg(desc_params, arg_max_log_files); command_line::add_arg(desc_params, arg_max_concurrency); command_line::add_arg(desc_params, arg_config_file); @@ -180,11 +185,15 @@ namespace wallet_args log_path = command_line::get_arg(vm, arg_log_file); else log_path = mlog_get_default_log_path(default_log_name); - mlog_configure(log_path, log_to_console, command_line::get_arg(vm, arg_max_log_file_size)); + mlog_configure(log_path, log_to_console, command_line::get_arg(vm, arg_max_log_file_size), command_line::get_arg(vm, arg_max_log_files)); if (!command_line::is_arg_defaulted(vm, arg_log_level)) { mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); } + else if (!log_to_console) + { + mlog_set_categories(""); + } if (notice) Print(print) << notice << ENDL; diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 214d51cde..b3141985d 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -53,6 +53,7 @@ namespace tools // wallet_not_initialized // multisig_export_needed // multisig_import_needed + // password_needed // std::logic_error // wallet_logic_error * // file_exists @@ -70,8 +71,10 @@ namespace tools // get_out_indexes_error // tx_parse_error // get_tx_pool_error + // out_of_hashchain_bounds_error + // signature_check_failed // transfer_error * - // get_random_outs_general_error + // get_outs_general_error // not_enough_unlocked_money // not_enough_money // tx_not_possible @@ -126,7 +129,7 @@ namespace tools get_blocks_error_message_index, get_hashes_error_message_index, get_out_indices_error_message_index, - get_random_outs_error_message_index + get_outs_error_message_index }; template<typename Base, int msg_index> @@ -208,6 +211,14 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct password_needed : public wallet_runtime_error + { + explicit password_needed(std::string&& loc, const std::string &msg = "Password needed") + : wallet_runtime_error(std::move(loc), msg) + { + } + }; + //---------------------------------------------------------------------------------------------------- const char* const file_error_messages[] = { "file already exists", "file not found", @@ -398,6 +409,24 @@ namespace tools std::string to_string() const { return refresh_error::to_string(); } }; //---------------------------------------------------------------------------------------------------- + struct out_of_hashchain_bounds_error : public refresh_error + { + explicit out_of_hashchain_bounds_error(std::string&& loc) + : refresh_error(std::move(loc), "Index out of bounds of of hashchain") + { + } + + std::string to_string() const { return refresh_error::to_string(); } + }; + //---------------------------------------------------------------------------------------------------- + struct signature_check_failed : public wallet_logic_error + { + explicit signature_check_failed(std::string&& loc, const std::string& message) + : wallet_logic_error(std::move(loc), "Signature check failed " + message) + { + } + }; + //---------------------------------------------------------------------------------------------------- struct transfer_error : public wallet_logic_error { protected: @@ -407,7 +436,7 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- - typedef failed_rpc_request<transfer_error, get_random_outs_error_message_index> get_random_outs_error; + typedef failed_rpc_request<transfer_error, get_outs_error_message_index> get_outs_error; //---------------------------------------------------------------------------------------------------- struct not_enough_unlocked_money : public transfer_error { @@ -659,30 +688,30 @@ namespace tools //---------------------------------------------------------------------------------------------------- struct tx_too_big : public transfer_error { - explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_size_limit) + explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_weight_limit) : transfer_error(std::move(loc), "transaction is too big") , m_tx(tx) - , m_tx_size_limit(tx_size_limit) + , m_tx_weight_limit(tx_weight_limit) { } const cryptonote::transaction& tx() const { return m_tx; } - uint64_t tx_size_limit() const { return m_tx_size_limit; } + uint64_t tx_weight_limit() const { return m_tx_weight_limit; } std::string to_string() const { std::ostringstream ss; cryptonote::transaction tx = m_tx; ss << transfer_error::to_string() << - ", tx_size_limit = " << m_tx_size_limit << - ", tx size = " << get_object_blobsize(m_tx) << + ", tx_weight_limit = " << m_tx_weight_limit << + ", tx weight = " << get_transaction_weight(m_tx) << ", tx:\n" << cryptonote::obj_to_json_str(tx); return ss.str(); } private: cryptonote::transaction m_tx; - uint64_t m_tx_size_limit; + uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- struct zero_destination : public transfer_error diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index dc1beef7b..5e6100dfd 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -51,6 +51,7 @@ using namespace epee; #include "mnemonics/electrum-words.h" #include "rpc/rpc_args.h" #include "rpc/core_rpc_server_commands_defs.h" +#include "daemonizer/daemonizer.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.rpc" @@ -59,7 +60,7 @@ namespace { const command_line::arg_descriptor<std::string, true> arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"}; const command_line::arg_descriptor<bool> arg_disable_rpc_login = {"disable-rpc-login", "Disable HTTP authentication for RPC connections served by this process"}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", "Enable commands which rely on a trusted daemon", false}; + const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false}; const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"}; const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false}; @@ -74,6 +75,21 @@ namespace } return pwd_container; } + //------------------------------------------------------------------------------------------------------------------------------ + void set_confirmations(tools::wallet_rpc::transfer_entry &entry, uint64_t blockchain_height, uint64_t block_reward) + { + if (entry.height >= blockchain_height) + { + entry.confirmations = 0; + entry.suggested_confirmations_threshold = 0; + return; + } + entry.confirmations = blockchain_height - entry.height; + if (block_reward == 0) + entry.suggested_confirmations_threshold = 0; + else + entry.suggested_confirmations_threshold = (entry.amount + block_reward - 1) / block_reward; + } } namespace tools @@ -84,7 +100,7 @@ namespace tools } //------------------------------------------------------------------------------------------------------------------------------ - wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_trusted_daemon(false), m_vm(NULL) + wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_restricted(false), m_vm(NULL) { } //------------------------------------------------------------------------------------------------------------------------------ @@ -104,7 +120,7 @@ namespace tools m_stop = false; m_net_server.add_idle_handler([this](){ try { - if (m_wallet) m_wallet->refresh(); + if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon()); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } @@ -140,30 +156,18 @@ namespace tools return false; m_vm = vm; - tools::wallet2 *walvars; - std::unique_ptr<tools::wallet2> tmpwal; - if (m_wallet) - walvars = m_wallet; - else - { - tmpwal = tools::wallet2::make_dummy(*m_vm, password_prompter); - walvars = tmpwal.get(); - } boost::optional<epee::net_utils::http::login> http_login{}; std::string bind_port = command_line::get_arg(*m_vm, arg_rpc_bind_port); const bool disable_auth = command_line::get_arg(*m_vm, arg_disable_rpc_login); - m_trusted_daemon = command_line::get_arg(*m_vm, arg_trusted_daemon); - if (!command_line::has_arg(*m_vm, arg_trusted_daemon)) + m_restricted = command_line::get_arg(*m_vm, arg_restricted); + if (!command_line::is_arg_defaulted(*m_vm, arg_wallet_dir)) { - if (tools::is_local_address(walvars->get_daemon_address())) + if (!command_line::is_arg_defaulted(*m_vm, wallet_args::arg_wallet_file())) { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; + MERROR(arg_wallet_dir.name << " and " << wallet_args::arg_wallet_file().name << " are incompatible, use only one of them"); + return false; } - } - if (command_line::has_arg(*m_vm, arg_wallet_dir)) - { m_wallet_dir = command_line::get_arg(*m_vm, arg_wallet_dir); #ifdef _WIN32 #define MKDIR(path, mode) mkdir(path) @@ -255,9 +259,10 @@ namespace tools entry.unlock_time = pd.m_unlock_time; entry.fee = pd.m_fee; entry.note = m_wallet->get_tx_note(pd.m_tx_hash); - entry.type = "in"; + entry.type = pd.m_coinbase ? "block" : "in"; entry.subaddr_index = pd.m_subaddr_index; entry.address = m_wallet->get_subaddress_as_str(pd.m_subaddr_index); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &txid, const tools::wallet2::confirmed_transfer_details &pd) @@ -284,6 +289,7 @@ namespace tools entry.type = "out"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &txid, const tools::wallet2::unconfirmed_transfer_details &pd) @@ -303,6 +309,7 @@ namespace tools entry.type = is_failed ? "failed" : "pending"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &payment_id, const tools::wallet2::pool_payment_details &ppd) @@ -322,6 +329,7 @@ namespace tools entry.type = "pool"; entry.subaddr_index = pd.m_subaddr_index; entry.address = m_wallet->get_subaddress_as_str(pd.m_subaddr_index); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er) @@ -336,14 +344,20 @@ namespace tools std::map<uint32_t, uint64_t> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(req.account_index); std::vector<tools::wallet2::transfer_details> transfers; m_wallet->get_transfers(transfers); - for (const auto& i : balance_per_subaddress) + std::set<uint32_t> address_indices = req.address_indices; + if (address_indices.empty()) + { + for (const auto& i : balance_per_subaddress) + address_indices.insert(i.first); + } + for (uint32_t i : address_indices) { wallet_rpc::COMMAND_RPC_GET_BALANCE::per_subaddress_info info; - info.address_index = i.first; + info.address_index = i; cryptonote::subaddress_index index = {req.account_index, info.address_index}; info.address = m_wallet->get_subaddress_as_str(index); - info.balance = i.second; - info.unlocked_balance = unlocked_balance_per_subaddress[i.first]; + info.balance = balance_per_subaddress[i]; + info.unlocked_balance = unlocked_balance_per_subaddress[i]; info.label = m_wallet->get_subaddress_label(index); info.num_unspent_outputs = std::count_if(transfers.begin(), transfers.end(), [&](const tools::wallet2::transfer_details& td) { return !td.m_spent && td.m_subaddr_index == index; }); res.per_subaddress.push_back(info); @@ -362,6 +376,7 @@ namespace tools if (!m_wallet) return not_open(er); try { + THROW_WALLET_EXCEPTION_IF(req.account_index >= m_wallet->get_num_subaddress_accounts(), error::account_index_outofbound); res.addresses.clear(); std::vector<uint32_t> req_address_index; if (req.address_index.empty()) @@ -377,6 +392,7 @@ namespace tools m_wallet->get_transfers(transfers); for (uint32_t i : req_address_index) { + THROW_WALLET_EXCEPTION_IF(i >= m_wallet->get_num_subaddresses(req.account_index), error::address_index_outofbound); res.addresses.resize(res.addresses.size() + 1); auto& info = res.addresses.back(); const cryptonote::subaddress_index index = {req.account_index, i}; @@ -395,6 +411,27 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + cryptonote::address_parse_info info; + if(!get_account_address_from_str(info, m_wallet->nettype(), req.address)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Invalid address"; + return false; + } + auto index = m_wallet->get_subaddress_index(info.address); + if (!index) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Address doesn't belong to the wallet"; + return false; + } + res.index = *index; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); @@ -500,6 +537,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); const std::pair<std::map<std::string, std::string>, std::vector<std::string>> account_tags = m_wallet->get_account_tags(); for (const std::pair<std::string, std::string>& p : account_tags.first) { @@ -518,6 +556,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); try { m_wallet->set_account_tag(req.accounts, req.tag); @@ -532,6 +571,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); try { m_wallet->set_account_tag(req.accounts, ""); @@ -546,6 +586,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); try { m_wallet->set_account_tag_description(req.tag, req.description); @@ -712,17 +753,17 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ template<typename Ts, typename Tu> bool wallet_rpc_server::fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er) { for (const auto & ptx : ptx_vector) { if (get_tx_key) { - std::string s = epee::string_tools::pod_to_hex(ptx.tx_key); + epee::wipeable_string s = epee::to_hex::wipeable_string(ptx.tx_key); for (const crypto::secret_key& additional_tx_key : ptx.additional_tx_keys) - s += epee::string_tools::pod_to_hex(additional_tx_key); - fill(tx_key, s); + s += epee::to_hex::wipeable_string(additional_tx_key); + fill(tx_key, std::string(s.data(), s.size())); } // Compute amount leaving wallet in tx. By convention dests does not include change outputs fill(amount, total_amount(ptx)); @@ -741,7 +782,16 @@ namespace tools } else { - if (!do_not_relay) + if (m_wallet->watch_only()){ + unsigned_txset = epee::string_tools::buff_to_hex_nodelimer(m_wallet->dump_tx_to_str(ptx_vector)); + if (unsigned_txset.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to save unsigned tx set after creation"; + return false; + } + } + else if (!do_not_relay) m_wallet->commit_tx(ptx_vector); // populate response with tx hashes @@ -769,7 +819,7 @@ namespace tools LOG_PRINT_L3("on_transfer starts"); if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -794,7 +844,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); if (ptx_vector.empty()) { @@ -811,7 +861,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); } catch (const std::exception& e) @@ -829,7 +879,7 @@ namespace tools std::vector<uint8_t> extra; if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -855,10 +905,10 @@ namespace tools } uint32_t priority = m_wallet->adjust_priority(req.priority); LOG_PRINT_L2("on_transfer_split calling create_transactions_2"); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); LOG_PRINT_L2("on_transfer_split called create_transactions_2"); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -869,10 +919,312 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + if(m_wallet->watch_only()) + { + er.code = WALLET_RPC_ERROR_CODE_WATCH_ONLY; + er.message = "command not supported by watch-only wallet"; + return false; + } + + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + + tools::wallet2::unsigned_tx_set exported_txs; + if(!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "cannot load unsigned_txset"; + return false; + } + + std::vector<tools::wallet2::pending_tx> ptxs; + try + { + tools::wallet2::signed_tx_set signed_txs; + std::string ciphertext = m_wallet->sign_tx_dump_to_str(exported_txs, ptxs, signed_txs); + if (ciphertext.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED; + er.message = "Failed to sign unsigned tx"; + return false; + } + + res.signed_txset = epee::string_tools::buff_to_hex_nodelimer(ciphertext); + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED; + er.message = std::string("Failed to sign unsigned tx: ") + e.what(); + return false; + } + + for (auto &ptx: ptxs) + { + res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); + if (req.get_tx_keys) + res.tx_key_list.push_back(epee::string_tools::pod_to_hex(ptx.tx_key)); + } + + if (req.export_raw) + { + for (auto &ptx: ptxs) + { + res.tx_raw_list.push_back(epee::string_tools::buff_to_hex_nodelimer(cryptonote::tx_to_blob(ptx.tx))); + } + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + if(m_wallet->watch_only()) + { + er.code = WALLET_RPC_ERROR_CODE_WATCH_ONLY; + er.message = "command not supported by watch-only wallet"; + return false; + } + + tools::wallet2::unsigned_tx_set exported_txs; + try + { + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + if(!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "cannot load unsigned_txset"; + return false; + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "failed to parse unsigned transfers: " + std::string(e.what()); + return false; + } + + std::vector<tools::wallet2::pending_tx> ptx; + try + { + // gather info to ask the user + std::unordered_map<cryptonote::account_public_address, std::pair<std::string, uint64_t>> dests; + int first_known_non_zero_change_index = -1; + for (size_t n = 0; n < exported_txs.txes.size(); ++n) + { + const tools::wallet2::tx_construction_data &cd = exported_txs.txes[n]; + res.desc.push_back({0, 0, std::numeric_limits<uint32_t>::max(), 0, {}, "", 0, "", 0, 0, ""}); + wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::transfer_description &desc = res.desc.back(); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + bool has_encrypted_payment_id = false; + crypto::hash8 payment_id8 = crypto::null_hash8; + if (cryptonote::parse_tx_extra(cd.extra, tx_extra_fields)) + { + cryptonote::tx_extra_nonce extra_nonce; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + crypto::hash payment_id; + if(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + { + desc.payment_id = epee::string_tools::pod_to_hex(payment_id8); + has_encrypted_payment_id = true; + } + else if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + desc.payment_id = epee::string_tools::pod_to_hex(payment_id); + } + } + } + + for (size_t s = 0; s < cd.sources.size(); ++s) + { + desc.amount_in += cd.sources[s].amount; + size_t ring_size = cd.sources[s].outputs.size(); + if (ring_size < desc.ring_size) + desc.ring_size = ring_size; + } + for (size_t d = 0; d < cd.splitted_dsts.size(); ++d) + { + const cryptonote::tx_destination_entry &entry = cd.splitted_dsts[d]; + std::string address = cryptonote::get_account_address_as_str(m_wallet->nettype(), entry.is_subaddress, entry.addr); + if (has_encrypted_payment_id && !entry.is_subaddress) + address = cryptonote::get_account_integrated_address_as_str(m_wallet->nettype(), entry.addr, payment_id8); + auto i = dests.find(entry.addr); + if (i == dests.end()) + dests.insert(std::make_pair(entry.addr, std::make_pair(address, entry.amount))); + else + i->second.second += entry.amount; + desc.amount_out += entry.amount; + } + if (cd.change_dts.amount > 0) + { + auto it = dests.find(cd.change_dts.addr); + if (it == dests.end()) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "Claimed change does not go to a paid address"; + return false; + } + if (it->second.second < cd.change_dts.amount) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "Claimed change is larger than payment to the change address"; + return false; + } + if (cd.change_dts.amount > 0) + { + if (first_known_non_zero_change_index == -1) + first_known_non_zero_change_index = n; + const tools::wallet2::tx_construction_data &cdn = exported_txs.txes[first_known_non_zero_change_index]; + if (memcmp(&cd.change_dts.addr, &cdn.change_dts.addr, sizeof(cd.change_dts.addr))) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "Change goes to more than one address"; + return false; + } + } + desc.change_amount += cd.change_dts.amount; + it->second.second -= cd.change_dts.amount; + if (it->second.second == 0) + dests.erase(cd.change_dts.addr); + } + + size_t n_dummy_outputs = 0; + for (auto i = dests.begin(); i != dests.end(); ) + { + if (i->second.second > 0) + { + desc.recipients.push_back({i->second.first, i->second.second}); + } + else + ++desc.dummy_outputs; + ++i; + } + + if (desc.change_amount > 0) + { + const tools::wallet2::tx_construction_data &cd0 = exported_txs.txes[0]; + desc.change_address = get_account_address_as_str(m_wallet->nettype(), cd0.subaddr_account > 0, cd0.change_dts.addr); + } + + desc.fee = desc.amount_in - desc.amount_out; + desc.unlock_time = cd.unlock_time; + desc.extra = epee::to_hex::string({cd.extra.data(), cd.extra.size()}); + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "failed to parse unsigned transfers"; + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.tx_data_hex, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + + std::vector<tools::wallet2::pending_tx> ptx_vector; + try + { + bool r = m_wallet->parse_tx_from_str(blob, ptx_vector, NULL); + if (!r) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_SIGNED_TX_DATA; + er.message = "Failed to parse signed tx data."; + return false; + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_SIGNED_TX_DATA; + er.message = std::string("Failed to parse signed tx: ") + e.what(); + return false; + } + + try + { + for (auto &ptx: ptx_vector) + { + m_wallet->commit_tx(ptx); + res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_SIGNED_SUBMISSION; + er.message = std::string("Failed to submit signed tx: ") + e.what(); + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -881,9 +1233,9 @@ namespace tools try { - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -900,7 +1252,7 @@ namespace tools std::vector<uint8_t> extra; if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -917,6 +1269,13 @@ namespace tools return false; } + if (req.outputs < 1) + { + er.code = WALLET_RPC_ERROR_CODE_TX_NOT_POSSIBLE; + er.message = "Amount of outputs should be greater than 0."; + return false; + } + try { uint64_t mixin; @@ -929,9 +1288,9 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, req.outputs, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -948,13 +1307,20 @@ namespace tools std::vector<uint8_t> extra; if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; return false; } + if (req.outputs < 1) + { + er.code = WALLET_RPC_ERROR_CODE_TX_NOT_POSSIBLE; + er.message = "Amount of outputs should be greater than 0."; + return false; + } + // validate the transfer requested and populate dsts & extra std::list<wallet_rpc::transfer_destination> destination; destination.push_back(wallet_rpc::transfer_destination()); @@ -985,7 +1351,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, req.outputs, mixin, req.unlock_time, priority, extra); if (ptx_vector.empty()) { @@ -1007,7 +1373,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); } catch (const std::exception& e) @@ -1086,7 +1452,39 @@ namespace tools } } - res.integrated_address = m_wallet->get_integrated_address_as_str(payment_id); + if (req.standard_address.empty()) + { + res.integrated_address = m_wallet->get_integrated_address_as_str(payment_id); + } + else + { + cryptonote::address_parse_info info; + if(!get_account_address_from_str(info, m_wallet->nettype(), req.standard_address)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Invalid address"; + return false; + } + if (info.is_subaddress) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Subaddress shouldn't be used"; + return false; + } + if (info.has_payment_id) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Already integrated address"; + return false; + } + if (req.payment_id.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment ID shouldn't be left unspecified"; + return false; + } + res.integrated_address = get_account_integrated_address_as_str(m_wallet->nettype(), info.address, payment_id); + } res.payment_id = epee::string_tools::pod_to_hex(payment_id); return true; } @@ -1132,7 +1530,7 @@ namespace tools bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1322,9 +1720,8 @@ namespace tools rpc_transfers.spent = td.m_spent; rpc_transfers.global_index = td.m_global_output_index; rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(td.m_txid); - rpc_transfers.tx_size = txBlob.size(); - rpc_transfers.subaddr_index = td.m_subaddr_index.minor; - rpc_transfers.key_image = req.verbose && td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; + rpc_transfers.subaddr_index = {td.m_subaddr_index.major, td.m_subaddr_index.minor}; + rpc_transfers.key_image = td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; res.transfers.push_back(rpc_transfers); } } @@ -1335,7 +1732,7 @@ namespace tools bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1344,19 +1741,24 @@ namespace tools if (req.key_type.compare("mnemonic") == 0) { - if (!m_wallet->get_seed(res.key)) + epee::wipeable_string seed; + if (!m_wallet->get_seed(seed)) { + er.code = WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC; er.message = "The wallet is non-deterministic. Cannot display seed."; return false; } + res.key = std::string(seed.data(), seed.size()); // send to the network, then wipe RAM :D } else if(req.key_type.compare("view_key") == 0) { - res.key = string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key); + epee::wipeable_string key = epee::to_hex::wipeable_string(m_wallet->get_account().get_keys().m_view_secret_key); + res.key = std::string(key.data(), key.size()); } else if(req.key_type.compare("spend_key") == 0) { - res.key = string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_spend_secret_key); + epee::wipeable_string key = epee::to_hex::wipeable_string(m_wallet->get_account().get_keys().m_spend_secret_key); + res.key = std::string(key.data(), key.size()); } else { @@ -1370,7 +1772,7 @@ namespace tools bool wallet_rpc_server::on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1392,7 +1794,7 @@ namespace tools bool wallet_rpc_server::on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1406,7 +1808,7 @@ namespace tools bool wallet_rpc_server::on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1441,7 +1843,7 @@ namespace tools bool wallet_rpc_server::on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1464,7 +1866,7 @@ namespace tools bool wallet_rpc_server::on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1536,7 +1938,7 @@ namespace tools bool wallet_rpc_server::on_set_attribute(const wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1551,7 +1953,7 @@ namespace tools bool wallet_rpc_server::on_get_attribute(const wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1582,11 +1984,11 @@ namespace tools return false; } - std::ostringstream oss; - oss << epee::string_tools::pod_to_hex(tx_key); + epee::wipeable_string s; + s += epee::to_hex::wipeable_string(tx_key); for (size_t i = 0; i < additional_tx_keys.size(); ++i) - oss << epee::string_tools::pod_to_hex(additional_tx_keys[i]); - res.tx_key = oss.str(); + s += epee::to_hex::wipeable_string(additional_tx_keys[i]); + res.tx_key = std::string(s.data(), s.size()); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1602,26 +2004,33 @@ namespace tools return false; } - std::string tx_key_str = req.tx_key; + epee::wipeable_string tx_key_str = req.tx_key; + if (tx_key_str.size() < 64 || tx_key_str.size() % 64) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY; + er.message = "Tx key has invalid format"; + return false; + } + const char *data = tx_key_str.data(); crypto::secret_key tx_key; - if (!epee::string_tools::hex_to_pod(tx_key_str.substr(0, 64), tx_key)) + if (!epee::wipeable_string(data, 64).hex_to_pod(unwrap(unwrap(tx_key)))) { er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY; er.message = "Tx key has invalid format"; return false; } - tx_key_str = tx_key_str.substr(64); + size_t offset = 64; std::vector<crypto::secret_key> additional_tx_keys; - while (!tx_key_str.empty()) + while (offset < tx_key_str.size()) { additional_tx_keys.resize(additional_tx_keys.size() + 1); - if (!epee::string_tools::hex_to_pod(tx_key_str.substr(0, 64), additional_tx_keys.back())) + if (!epee::wipeable_string(data + offset, 64).hex_to_pod(unwrap(unwrap(additional_tx_keys.back())))) { er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY; er.message = "Tx key has invalid format"; return false; } - tx_key_str = tx_key_str.substr(64); + offset += 64; } cryptonote::address_parse_info info; @@ -1827,7 +2236,7 @@ namespace tools bool wallet_rpc_server::on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1893,7 +2302,7 @@ namespace tools bool wallet_rpc_server::on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1932,8 +2341,8 @@ namespace tools for (std::list<std::pair<crypto::hash, tools::wallet2::payment_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { if (i->second.m_tx_hash == txid) { - fill_transfer_entry(res.transfer, i->second.m_tx_hash, i->first, i->second); - return true; + res.transfers.resize(res.transfers.size() + 1); + fill_transfer_entry(res.transfers.back(), i->second.m_tx_hash, i->first, i->second); } } @@ -1942,8 +2351,8 @@ namespace tools for (std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>>::const_iterator i = payments_out.begin(); i != payments_out.end(); ++i) { if (i->first == txid) { - fill_transfer_entry(res.transfer, i->first, i->second); - return true; + res.transfers.resize(res.transfers.size() + 1); + fill_transfer_entry(res.transfers.back(), i->first, i->second); } } @@ -1952,8 +2361,8 @@ namespace tools for (std::list<std::pair<crypto::hash, tools::wallet2::unconfirmed_transfer_details>>::const_iterator i = upayments.begin(); i != upayments.end(); ++i) { if (i->first == txid) { - fill_transfer_entry(res.transfer, i->first, i->second); - return true; + res.transfers.resize(res.transfers.size() + 1); + fill_transfer_entry(res.transfers.back(), i->first, i->second); } } @@ -1964,16 +2373,88 @@ namespace tools for (std::list<std::pair<crypto::hash, tools::wallet2::pool_payment_details>>::const_iterator i = pool_payments.begin(); i != pool_payments.end(); ++i) { if (i->second.m_pd.m_tx_hash == txid) { - fill_transfer_entry(res.transfer, i->first, i->second); - return true; + res.transfers.resize(res.transfers.size() + 1); + fill_transfer_entry(res.transfers.back(), i->first, i->second); } } + if (!res.transfers.empty()) + { + res.transfer = res.transfers.front(); // backward compat + return true; + } + er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID; er.message = "Transaction not found."; return false; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + + try + { + res.outputs_data_hex = epee::string_tools::buff_to_hex_nodelimer(m_wallet->export_outputs_to_str()); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.outputs_data_hex, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + + try + { + res.num_imported = m_wallet->import_outputs_from_str(blob); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); @@ -2000,13 +2481,13 @@ namespace tools bool wallet_rpc_server::on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; return false; } - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2054,6 +2535,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); std::string error; std::string uri = m_wallet->make_uri(req.address, req.payment_id, req.amount, req.tx_description, req.recipient_name, error); if (uri.empty()) @@ -2110,7 +2592,7 @@ namespace tools bool wallet_rpc_server::on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2185,7 +2667,7 @@ namespace tools bool wallet_rpc_server::on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2208,10 +2690,32 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + try + { + m_wallet->refresh(m_wallet->is_trusted_daemon(), req.start_height, res.blocks_fetched, res.received_money); + return true; + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2233,7 +2737,7 @@ namespace tools bool wallet_rpc_server::on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2267,6 +2771,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); cryptonote::COMMAND_RPC_STOP_MINING::request daemon_req; cryptonote::COMMAND_RPC_STOP_MINING::response daemon_res; bool r = m_wallet->invoke_http_json("/stop_mining", daemon_req, daemon_res); @@ -2338,7 +2843,7 @@ namespace tools command_line::add_arg(desc, arg_password); po::store(po::parse_command_line(argc, argv, desc), vm2); } - std::unique_ptr<tools::wallet2> wal = tools::wallet2::make_new(vm2, nullptr).first; + std::unique_ptr<tools::wallet2> wal = tools::wallet2::make_new(vm2, true, nullptr).first; if (!wal) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -2366,8 +2871,20 @@ namespace tools er.message = "Failed to generate wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } @@ -2412,7 +2929,7 @@ namespace tools } std::unique_ptr<tools::wallet2> wal = nullptr; try { - wal = tools::wallet2::make_from_file(vm2, wallet_file, nullptr).first; + wal = tools::wallet2::make_from_file(vm2, true, wallet_file, nullptr).first; } catch (const std::exception& e) { @@ -2424,12 +2941,73 @@ namespace tools er.message = "Failed to open wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + m_wallet = NULL; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->verify_password(req.old_password)) + { + try + { + m_wallet->change_password(m_wallet->get_wallet_file(), req.old_password, req.new_password); + LOG_PRINT_L0("Wallet password changed."); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + } + else + { + er.code = WALLET_RPC_ERROR_CODE_INVALID_PASSWORD; + er.message = "Invalid original password."; + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code) { try { @@ -2495,6 +3073,11 @@ namespace tools er.code = WALLET_RPC_ERROR_CODE_ADDRESS_INDEX_OUT_OF_BOUNDS; er.message = e.what(); } + catch (const error::signature_check_failed& e) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_SIGNATURE; + er.message = e.what(); + } catch (const std::exception& e) { er.code = default_error_code; @@ -2517,7 +3100,7 @@ namespace tools bool wallet_rpc_server::on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2543,7 +3126,7 @@ namespace tools bool wallet_rpc_server::on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2580,7 +3163,7 @@ namespace tools bool wallet_rpc_server::on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2620,7 +3203,7 @@ namespace tools bool wallet_rpc_server::on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2671,7 +3254,7 @@ namespace tools return false; } - if (m_trusted_daemon) + if (m_wallet->is_trusted_daemon()) { try { @@ -2693,7 +3276,7 @@ namespace tools bool wallet_rpc_server::on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2714,7 +3297,7 @@ namespace tools return false; } - if (req.multisig_info.size() < threshold - 1) + if (req.multisig_info.size() < 1 || req.multisig_info.size() > total) { er.code = WALLET_RPC_ERROR_CODE_THRESHOLD_NOT_REACHED; er.message = "Needs multisig info from more participants"; @@ -2741,10 +3324,59 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + bool ready; + uint32_t threshold, total; + if (!m_wallet->multisig(&ready, &threshold, &total)) + { + er.code = WALLET_RPC_ERROR_CODE_NOT_MULTISIG; + er.message = "This wallet is not multisig"; + return false; + } + + if (ready) + { + er.code = WALLET_RPC_ERROR_CODE_ALREADY_MULTISIG; + er.message = "This wallet is multisig, and already finalized"; + return false; + } + + if (req.multisig_info.size() < 1 || req.multisig_info.size() > total) + { + er.code = WALLET_RPC_ERROR_CODE_THRESHOLD_NOT_REACHED; + er.message = "Needs multisig info from more participants"; + return false; + } + + try + { + res.multisig_info = m_wallet->exchange_multisig_keys(req.password, req.multisig_info); + if (res.multisig_info.empty()) + { + res.address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = std::string("Error calling exchange_multisig_info: ") + e.what(); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2813,7 +3445,7 @@ namespace tools bool wallet_rpc_server::on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2884,23 +3516,198 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ } +class t_daemon +{ +private: + const boost::program_options::variables_map& vm; + + std::unique_ptr<tools::wallet_rpc_server> wrpc; + +public: + t_daemon(boost::program_options::variables_map const & _vm) + : vm(_vm) + , wrpc(new tools::wallet_rpc_server) + { + } + + bool run() + { + std::unique_ptr<tools::wallet2> wal; + try + { + const bool testnet = tools::wallet2::has_testnet_option(vm); + const bool stagenet = tools::wallet2::has_stagenet_option(vm); + if (testnet && stagenet) + { + MERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --testnet and --stagenet")); + return false; + } + + const auto arg_wallet_file = wallet_args::arg_wallet_file(); + const auto arg_from_json = wallet_args::arg_generate_from_json(); + + const auto wallet_file = command_line::get_arg(vm, arg_wallet_file); + const auto from_json = command_line::get_arg(vm, arg_from_json); + const auto wallet_dir = command_line::get_arg(vm, arg_wallet_dir); + const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password); + const auto password_prompt = prompt_for_password ? password_prompter : nullptr; + + if(!wallet_file.empty() && !from_json.empty()) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --wallet-file and --generate-from-json")); + return false; + } + + if (!wallet_dir.empty()) + { + wal = NULL; + goto just_dir; + } + + if (wallet_file.empty() && from_json.empty()) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Must specify --wallet-file or --generate-from-json or --wallet-dir")); + return false; + } + + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Loading wallet...")); + if(!wallet_file.empty()) + { + wal = tools::wallet2::make_from_file(vm, true, wallet_file, password_prompt).first; + } + else + { + try + { + auto rc = tools::wallet2::make_from_json(vm, true, from_json, password_prompt); + wal = std::move(rc.first); + } + catch (const std::exception &e) + { + MERROR("Error creating wallet: " << e.what()); + return false; + } + } + if (!wal) + { + return false; + } + + bool quit = false; + tools::signal_handler::install([&wal, &quit](int) { + assert(wal); + quit = true; + wal->stop(); + }); + + wal->refresh(wal->is_trusted_daemon()); + // if we ^C during potentially length load/refresh, there's no server loop yet + if (quit) + { + MINFO(tools::wallet_rpc_server::tr("Saving wallet...")); + wal->store(); + MINFO(tools::wallet_rpc_server::tr("Successfully saved")); + return false; + } + MINFO(tools::wallet_rpc_server::tr("Successfully loaded")); + } + catch (const std::exception& e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Wallet initialization failed: ") << e.what()); + return false; + } + just_dir: + if (wal) wrpc->set_wallet(wal.release()); + bool r = wrpc->init(&vm); + CHECK_AND_ASSERT_MES(r, false, tools::wallet_rpc_server::tr("Failed to initialize wallet RPC server")); + tools::signal_handler::install([this](int) { + wrpc->send_stop_signal(); + }); + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Starting wallet RPC server")); + try + { + wrpc->run(); + } + catch (const std::exception &e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Failed to run wallet: ") << e.what()); + return false; + } + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Stopped wallet RPC server")); + try + { + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Saving wallet...")); + wrpc->stop(); + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Successfully saved")); + } + catch (const std::exception& e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Failed to save wallet: ") << e.what()); + return false; + } + return true; + } + + void stop() + { + wrpc->send_stop_signal(); + } +}; + +class t_executor final +{ +public: + static std::string const NAME; + + typedef ::t_daemon t_daemon; + + std::string const & name() const + { + return NAME; + } + + t_daemon create_daemon(boost::program_options::variables_map const & vm) + { + return t_daemon(vm); + } + + bool run_non_interactive(boost::program_options::variables_map const & vm) + { + return t_daemon(vm).run(); + } + + bool run_interactive(boost::program_options::variables_map const & vm) + { + return t_daemon(vm).run(); + } +}; + +std::string const t_executor::NAME = "Wallet RPC Daemon"; + int main(int argc, char** argv) { + TRY_ENTRY(); + namespace po = boost::program_options; const auto arg_wallet_file = wallet_args::arg_wallet_file(); const auto arg_from_json = wallet_args::arg_generate_from_json(); + po::options_description hidden_options("Hidden"); + po::options_description desc_params(wallet_args::tr("Wallet options")); tools::wallet2::init_options(desc_params); command_line::add_arg(desc_params, arg_rpc_bind_port); command_line::add_arg(desc_params, arg_disable_rpc_login); - command_line::add_arg(desc_params, arg_trusted_daemon); + command_line::add_arg(desc_params, arg_restricted); cryptonote::rpc_args::init_options(desc_params); command_line::add_arg(desc_params, arg_wallet_file); command_line::add_arg(desc_params, arg_from_json); command_line::add_arg(desc_params, arg_wallet_dir); command_line::add_arg(desc_params, arg_prompt_for_password); + daemonizer::init_options(hidden_options, desc_params); + desc_params.add(hidden_options); + boost::optional<po::variables_map> vm; bool should_terminate = false; std::tie(vm, should_terminate) = wallet_args::main( @@ -2922,115 +3729,6 @@ int main(int argc, char** argv) { return 0; } - std::unique_ptr<tools::wallet2> wal; - try - { - const bool testnet = tools::wallet2::has_testnet_option(*vm); - const bool stagenet = tools::wallet2::has_stagenet_option(*vm); - if (testnet && stagenet) - { - MERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --testnet and --stagenet")); - return 1; - } - - const auto wallet_file = command_line::get_arg(*vm, arg_wallet_file); - const auto from_json = command_line::get_arg(*vm, arg_from_json); - const auto wallet_dir = command_line::get_arg(*vm, arg_wallet_dir); - const auto prompt_for_password = command_line::get_arg(*vm, arg_prompt_for_password); - const auto password_prompt = prompt_for_password ? password_prompter : nullptr; - - if(!wallet_file.empty() && !from_json.empty()) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --wallet-file and --generate-from-json")); - return 1; - } - - if (!wallet_dir.empty()) - { - wal = NULL; - goto just_dir; - } - - if (wallet_file.empty() && from_json.empty()) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Must specify --wallet-file or --generate-from-json or --wallet-dir")); - return 1; - } - - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Loading wallet...")); - if(!wallet_file.empty()) - { - wal = tools::wallet2::make_from_file(*vm, wallet_file, password_prompt).first; - } - else - { - try - { - wal = tools::wallet2::make_from_json(*vm, from_json, password_prompt); - } - catch (const std::exception &e) - { - MERROR("Error creating wallet: " << e.what()); - return 1; - } - } - if (!wal) - { - return 1; - } - - bool quit = false; - tools::signal_handler::install([&wal, &quit](int) { - assert(wal); - quit = true; - wal->stop(); - }); - - wal->refresh(); - // if we ^C during potentially length load/refresh, there's no server loop yet - if (quit) - { - MINFO(tools::wallet_rpc_server::tr("Saving wallet...")); - wal->store(); - MINFO(tools::wallet_rpc_server::tr("Successfully saved")); - return 1; - } - MINFO(tools::wallet_rpc_server::tr("Successfully loaded")); - } - catch (const std::exception& e) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Wallet initialization failed: ") << e.what()); - return 1; - } -just_dir: - tools::wallet_rpc_server wrpc; - if (wal) wrpc.set_wallet(wal.release()); - bool r = wrpc.init(&(vm.get())); - CHECK_AND_ASSERT_MES(r, 1, tools::wallet_rpc_server::tr("Failed to initialize wallet RPC server")); - tools::signal_handler::install([&wrpc](int) { - wrpc.send_stop_signal(); - }); - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Starting wallet RPC server")); - try - { - wrpc.run(); - } - catch (const std::exception &e) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Failed to run wallet: ") << e.what()); - return 1; - } - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Stopped wallet RPC server")); - try - { - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Saving wallet...")); - wrpc.stop(); - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Successfully saved")); - } - catch (const std::exception& e) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Failed to save wallet: ") << e.what()); - return 1; - } - return 0; + return daemonizer::daemonize(argc, const_cast<const char**>(argv), t_executor{}, *vm) ? 0 : 1; + CATCH_ENTRY_L0("main", 1); } diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index cb1a274b6..887723ed5 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -35,6 +35,7 @@ #include <string> #include "common/util.h" #include "net/http_server_impl_base.h" +#include "math_helper.h" #include "wallet_rpc_server_commands_defs.h" #include "wallet2.h" @@ -69,6 +70,7 @@ namespace tools BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC_WE("get_balance", on_getbalance, wallet_rpc::COMMAND_RPC_GET_BALANCE) MAP_JON_RPC_WE("get_address", on_getaddress, wallet_rpc::COMMAND_RPC_GET_ADDRESS) + MAP_JON_RPC_WE("get_address_index", on_getaddress_index, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX) MAP_JON_RPC_WE("getbalance", on_getbalance, wallet_rpc::COMMAND_RPC_GET_BALANCE) MAP_JON_RPC_WE("getaddress", on_getaddress, wallet_rpc::COMMAND_RPC_GET_ADDRESS) MAP_JON_RPC_WE("create_address", on_create_address, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS) @@ -84,6 +86,9 @@ namespace tools MAP_JON_RPC_WE("getheight", on_getheight, wallet_rpc::COMMAND_RPC_GET_HEIGHT) MAP_JON_RPC_WE("transfer", on_transfer, wallet_rpc::COMMAND_RPC_TRANSFER) MAP_JON_RPC_WE("transfer_split", on_transfer_split, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT) + MAP_JON_RPC_WE("sign_transfer", on_sign_transfer, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER) + MAP_JON_RPC_WE("describe_transfer", on_describe_transfer, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER) + MAP_JON_RPC_WE("submit_transfer", on_submit_transfer, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER) MAP_JON_RPC_WE("sweep_dust", on_sweep_dust, wallet_rpc::COMMAND_RPC_SWEEP_DUST) MAP_JON_RPC_WE("sweep_unmixable", on_sweep_dust, wallet_rpc::COMMAND_RPC_SWEEP_DUST) MAP_JON_RPC_WE("sweep_all", on_sweep_all, wallet_rpc::COMMAND_RPC_SWEEP_ALL) @@ -114,6 +119,8 @@ namespace tools MAP_JON_RPC_WE("get_transfer_by_txid", on_get_transfer_by_txid, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID) MAP_JON_RPC_WE("sign", on_sign, wallet_rpc::COMMAND_RPC_SIGN) MAP_JON_RPC_WE("verify", on_verify, wallet_rpc::COMMAND_RPC_VERIFY) + MAP_JON_RPC_WE("export_outputs", on_export_outputs, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS) + MAP_JON_RPC_WE("import_outputs", on_import_outputs, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS) MAP_JON_RPC_WE("export_key_images", on_export_key_images, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES) MAP_JON_RPC_WE("import_key_images", on_import_key_images, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES) MAP_JON_RPC_WE("make_uri", on_make_uri, wallet_rpc::COMMAND_RPC_MAKE_URI) @@ -121,18 +128,22 @@ namespace tools MAP_JON_RPC_WE("get_address_book", on_get_address_book, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("add_address_book", on_add_address_book, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) + MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) MAP_JON_RPC_WE("get_languages", on_get_languages, wallet_rpc::COMMAND_RPC_GET_LANGUAGES) MAP_JON_RPC_WE("create_wallet", on_create_wallet, wallet_rpc::COMMAND_RPC_CREATE_WALLET) MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) + MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) + MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) MAP_JON_RPC_WE("export_multisig_info", on_export_multisig, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG) MAP_JON_RPC_WE("import_multisig_info", on_import_multisig, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG) MAP_JON_RPC_WE("finalize_multisig", on_finalize_multisig, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG) + MAP_JON_RPC_WE("exchange_multisig_keys", on_exchange_multisig_keys, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS) MAP_JON_RPC_WE("sign_multisig", on_sign_multisig, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG) MAP_JON_RPC_WE("submit_multisig", on_submit_multisig, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG) MAP_JON_RPC_WE("get_version", on_get_version, wallet_rpc::COMMAND_RPC_GET_VERSION) @@ -142,6 +153,7 @@ namespace tools //json_rpc bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er); bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er); + bool on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er); bool on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er); bool on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er); bool on_get_accounts(const wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::response& res, epee::json_rpc::error& er); @@ -155,6 +167,9 @@ namespace tools bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er); bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er); + bool on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er); + bool on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er); + bool on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er); bool on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er); bool on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er); bool on_sweep_single(const wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::request& req, wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::response& res, epee::json_rpc::error& er); @@ -183,6 +198,8 @@ namespace tools bool on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er); bool on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er); bool on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er); + bool on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er); + bool on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er); bool on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er); bool on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er); bool on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er); @@ -190,18 +207,22 @@ namespace tools bool on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); + bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er); bool on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er); bool on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er); bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); + bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); + bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er); bool on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er); bool on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er); + bool on_exchange_multisig_keys(const wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::request& req, wallet_rpc::COMMAND_RPC_EXCHANGE_MULTISIG_KEYS::response& res, epee::json_rpc::error& er); bool on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er); bool on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er); bool on_get_version(const wallet_rpc::COMMAND_RPC_GET_VERSION::request& req, wallet_rpc::COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& er); @@ -219,14 +240,14 @@ namespace tools template<typename Ts, typename Tu> bool fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er); wallet2 *m_wallet; std::string m_wallet_dir; tools::private_file rpc_login_file; std::atomic<bool> m_stop; - bool m_trusted_daemon; + bool m_restricted; const boost::program_options::variables_map *m_vm; }; } diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index d44aa459f..924f3a0f1 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 0 +#define WALLET_RPC_VERSION_MINOR 5 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -62,8 +62,10 @@ namespace wallet_rpc struct request { uint32_t account_index; + std::set<uint32_t> address_indices; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) + KV_SERIALIZE(address_indices) END_KV_SERIALIZE_MAP() }; @@ -141,6 +143,25 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_GET_ADDRESS_INDEX + { + struct request + { + std::string address; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + cryptonote::subaddress_index index; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(index) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_CREATE_ADDRESS { struct request @@ -424,22 +445,22 @@ namespace wallet_rpc { std::string tx_hash; std::string tx_key; - std::list<std::string> amount_keys; uint64_t amount; uint64_t fee; std::string tx_blob; std::string tx_metadata; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(tx_key) - KV_SERIALIZE(amount_keys) KV_SERIALIZE(amount) KV_SERIALIZE(fee) KV_SERIALIZE(tx_blob) KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -495,6 +516,7 @@ namespace wallet_rpc std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -504,6 +526,120 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_DESCRIBE_TRANSFER + { + struct recipient + { + std::string address; + uint64_t amount; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(amount) + END_KV_SERIALIZE_MAP() + }; + + struct transfer_description + { + uint64_t amount_in; + uint64_t amount_out; + uint32_t ring_size; + uint64_t unlock_time; + std::list<recipient> recipients; + std::string payment_id; + uint64_t change_amount; + std::string change_address; + uint64_t fee; + uint32_t dummy_outputs; + std::string extra; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount_in) + KV_SERIALIZE(amount_out) + KV_SERIALIZE(ring_size) + KV_SERIALIZE(unlock_time) + KV_SERIALIZE(recipients) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(change_amount) + KV_SERIALIZE(change_address) + KV_SERIALIZE(fee) + KV_SERIALIZE(dummy_outputs) + KV_SERIALIZE(extra) + END_KV_SERIALIZE_MAP() + }; + + struct request + { + std::string unsigned_txset; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(unsigned_txset) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<transfer_description> desc; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(desc) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SIGN_TRANSFER + { + struct request + { + std::string unsigned_txset; + bool export_raw; + bool get_tx_keys; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE_OPT(export_raw, false) + KV_SERIALIZE_OPT(get_tx_keys, false) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string signed_txset; + std::list<std::string> tx_hash_list; + std::list<std::string> tx_raw_list; + std::list<std::string> tx_key_list; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(signed_txset) + KV_SERIALIZE(tx_hash_list) + KV_SERIALIZE(tx_raw_list) + KV_SERIALIZE(tx_key_list) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SUBMIT_TRANSFER + { + struct request + { + std::string tx_data_hex; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(tx_data_hex) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<std::string> tx_hash_list; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(tx_hash_list) END_KV_SERIALIZE_MAP() }; }; @@ -543,6 +679,7 @@ namespace wallet_rpc std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -552,6 +689,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -566,6 +704,7 @@ namespace wallet_rpc uint32_t priority; uint64_t mixin; uint64_t ring_size; + uint64_t outputs; uint64_t unlock_time; std::string payment_id; bool get_tx_keys; @@ -581,6 +720,7 @@ namespace wallet_rpc KV_SERIALIZE(priority) KV_SERIALIZE_OPT(mixin, (uint64_t)0) KV_SERIALIZE_OPT(ring_size, (uint64_t)0) + KV_SERIALIZE_OPT(outputs, (uint64_t)1) KV_SERIALIZE(unlock_time) KV_SERIALIZE(payment_id) KV_SERIALIZE(get_tx_keys) @@ -609,6 +749,7 @@ namespace wallet_rpc std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -618,6 +759,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -630,6 +772,7 @@ namespace wallet_rpc uint32_t priority; uint64_t mixin; uint64_t ring_size; + uint64_t outputs; uint64_t unlock_time; std::string payment_id; bool get_tx_key; @@ -643,6 +786,7 @@ namespace wallet_rpc KV_SERIALIZE(priority) KV_SERIALIZE_OPT(mixin, (uint64_t)0) KV_SERIALIZE_OPT(ring_size, (uint64_t)0) + KV_SERIALIZE_OPT(outputs, (uint64_t)1) KV_SERIALIZE(unlock_time) KV_SERIALIZE(payment_id) KV_SERIALIZE(get_tx_key) @@ -662,6 +806,7 @@ namespace wallet_rpc std::string tx_blob; std::string tx_metadata; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) @@ -671,6 +816,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob) KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -689,15 +835,9 @@ namespace wallet_rpc struct response { std::string tx_hash; - std::string tx_key; - uint64_t fee; - std::string tx_blob; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_key) - KV_SERIALIZE(fee) - KV_SERIALIZE(tx_blob) END_KV_SERIALIZE_MAP() }; }; @@ -788,8 +928,7 @@ namespace wallet_rpc bool spent; uint64_t global_index; std::string tx_hash; - uint64_t tx_size; - uint32_t subaddr_index; + cryptonote::subaddress_index subaddr_index; std::string key_image; BEGIN_KV_SERIALIZE_MAP() @@ -797,7 +936,6 @@ namespace wallet_rpc KV_SERIALIZE(spent) KV_SERIALIZE(global_index) KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_size) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) END_KV_SERIALIZE_MAP() @@ -810,13 +948,11 @@ namespace wallet_rpc std::string transfer_type; uint32_t account_index; std::set<uint32_t> subaddr_indices; - bool verbose; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(transfer_type) KV_SERIALIZE(account_index) KV_SERIALIZE(subaddr_indices) - KV_SERIALIZE(verbose) END_KV_SERIALIZE_MAP() }; @@ -856,9 +992,11 @@ namespace wallet_rpc { struct request { + std::string standard_address; std::string payment_id; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(standard_address) KV_SERIALIZE(payment_id) END_KV_SERIALIZE_MAP() }; @@ -1136,6 +1274,8 @@ namespace wallet_rpc cryptonote::subaddress_index subaddr_index; std::string address; bool double_spend_seen; + uint64_t confirmations; + uint64_t suggested_confirmations_threshold; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txid); @@ -1151,6 +1291,8 @@ namespace wallet_rpc KV_SERIALIZE(subaddr_index); KV_SERIALIZE(address); KV_SERIALIZE(double_spend_seen) + KV_SERIALIZE_OPT(confirmations, (uint64_t)0) + KV_SERIALIZE_OPT(suggested_confirmations_threshold, (uint64_t)0) END_KV_SERIALIZE_MAP() }; @@ -1322,9 +1464,11 @@ namespace wallet_rpc struct response { transfer_entry transfer; + std::list<transfer_entry> transfers; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(transfer); + KV_SERIALIZE(transfers); END_KV_SERIALIZE_MAP() }; }; @@ -1375,6 +1519,45 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_EXPORT_OUTPUTS + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string outputs_data_hex; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(outputs_data_hex); + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_IMPORT_OUTPUTS + { + struct request + { + std::string outputs_data_hex; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(outputs_data_hex); + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t num_imported; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(num_imported); + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_EXPORT_KEY_IMAGES { struct request @@ -1590,6 +1773,29 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_REFRESH + { + struct request + { + uint64_t start_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(start_height, (uint64_t) 0) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t blocks_fetched; + bool received_money; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(blocks_fetched); + KV_SERIALIZE(received_money); + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_START_MINING { struct request @@ -1684,6 +1890,40 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_CLOSE_WALLET + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_CHANGE_WALLET_PASSWORD + { + struct request + { + std::string old_password; + std::string new_password; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(old_password) + KV_SERIALIZE(new_password) + END_KV_SERIALIZE_MAP() + }; + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_IS_MULTISIG { struct request @@ -1815,6 +2055,31 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_EXCHANGE_MULTISIG_KEYS + { + struct request + { + std::string password; + std::vector<std::string> multisig_info; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(password) + KV_SERIALIZE(multisig_info) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string address; + std::string multisig_info; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(multisig_info) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_SIGN_MULTISIG { struct request diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index d47467940..9b3a2847d 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -69,3 +69,8 @@ #define WALLET_RPC_ERROR_CODE_MULTISIG_SUBMISSION -36 #define WALLET_RPC_ERROR_CODE_NOT_ENOUGH_UNLOCKED_MONEY -37 #define WALLET_RPC_ERROR_CODE_NO_DAEMON_CONNECTION -38 +#define WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA -39 +#define WALLET_RPC_ERROR_CODE_BAD_SIGNED_TX_DATA -40 +#define WALLET_RPC_ERROR_CODE_SIGNED_SUBMISSION -41 +#define WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED -42 +#define WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC -43 |