diff options
Diffstat (limited to 'src')
132 files changed, 9278 insertions, 5251 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index f827ab7c3..6c79120e8 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -224,7 +224,7 @@ struct Dbt_safe : public Dbt namespace cryptonote { -void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) +void BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -255,7 +255,7 @@ void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const if (res) throw0(DB_ERROR("Failed to add block blob to db transaction.")); - Dbt_copy<size_t> sz(block_size); + Dbt_copy<size_t> sz(block_weight); if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0)) throw0(DB_ERROR("Failed to add block size to db transaction.")); @@ -1353,7 +1353,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainBDB::get_block_size(const uint64_t& height) const +size_t BlockchainBDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1861,7 +1861,7 @@ void BlockchainBDB::block_txn_abort() // TODO } -uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) +uint64_t BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1874,7 +1874,7 @@ uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, co uint64_t num_outputs = m_num_outputs; try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); m_write_txn = NULL; TIME_MEASURE_START(time1); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index c90d030a2..76d0a0517 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -266,7 +266,7 @@ public: virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -303,7 +303,6 @@ public: virtual uint64_t get_indexing_base() const { return 1; } virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -319,7 +318,7 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -354,7 +353,7 @@ public: private: virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const crypto::hash& block_hash @@ -419,6 +418,7 @@ private: * @return the global index of the desired output */ uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); + output_data_t get_output_key(const uint64_t& global_index) const; void checkpoint_worker() const; void check_open() const; diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 8544cc3f0..be0ffeac3 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -87,8 +87,8 @@ const command_line::arg_descriptor<std::string> arg_db_type = { }; const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" -, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[nblocks_per_sync]." -, "fast:async:1000" +, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]." +, "fast:async:250000000bytes" }; const command_line::arg_descriptor<bool> arg_db_salvage = { "db-salvage" @@ -196,7 +196,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } uint64_t BlockchainDB::add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -241,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); + add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 6851e2404..396ae7544 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -137,7 +137,7 @@ struct txpool_tx_meta_t { crypto::hash max_used_block_id; crypto::hash last_failed_id; - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t max_used_block_height; uint64_t last_failed_height; @@ -358,13 +358,13 @@ private: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -376,7 +376,7 @@ private: * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -789,7 +789,7 @@ public: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the size of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block @@ -797,7 +797,7 @@ public: * @return the height of the chain post-addition */ virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -930,18 +930,18 @@ public: virtual uint64_t get_top_block_timestamp() const = 0; /** - * @brief fetch a block's size + * @brief fetch a block's weight * - * The subclass should return the size of the block with the + * The subclass should return the weight of the block with the * given height. * * If the block does not exist, the subclass should throw BLOCK_DNE * * @param height the height requested * - * @return the size + * @return the weight */ - virtual size_t get_block_size(const uint64_t& height) const = 0; + virtual size_t get_block_weight(const uint64_t& height) const = 0; /** * @brief fetch a block's cumulative difficulty @@ -1262,23 +1262,6 @@ public: virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0; /** - * @brief get some of an output's data - * - * The subclass should return the public key, unlock time, and block height - * for the output with the given global index, collected in a struct. - * - * If the output cannot be found, the subclass should throw OUTPUT_DNE. - * - * If any of these parts cannot be found, but some are, the subclass - * should throw DB_ERROR with a message stating as much. - * - * @param global_index the output's index (global) - * - * @return the requested output data - */ - virtual output_data_t get_output_key(const uint64_t& global_index) const = 0; - - /** * @brief gets an output's tx hash and index * * The subclass should return the hash of the transaction which created the diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 5d1679c0c..9e22e2e4b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -256,7 +256,7 @@ typedef struct mdb_block_info_old uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat + uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; } mdb_block_info_old; @@ -266,7 +266,7 @@ typedef struct mdb_block_info uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat + uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; @@ -545,8 +545,8 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const LOG_PRINT_L1("Space used: " << size_used); LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); LOG_PRINT_L1("Size threshold: " << threshold_size); - float resize_percent_old = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old); + float resize_percent = RESIZE_PERCENT; + LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { @@ -559,10 +559,6 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const return false; } - std::mt19937 engine(std::random_device{}()); - std::uniform_real_distribution<double> fdis(0.6, 0.9); - double resize_percent = fdis(engine); - if ((double)size_used / mei.me_mapsize > resize_percent) { LOG_PRINT_L1("Threshold met (percent-based)"); @@ -656,8 +652,11 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin block_rtxn_start(&rtxn, &rcurs); for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) { - uint32_t block_size = get_block_size(block_num); - total_block_size += block_size; + // we have access to block weight, which will be greater or equal to block size, + // so use this as a proxy. If it's too much off, we might have to check actual size, + // which involves reading more data, so is not really wanted + size_t block_weight = get_block_weight(block_num); + total_block_size += block_weight; // Track number of blocks being totalled here instead of assuming, in case // some blocks were to be skipped for being outliers. ++num_blocks_used; @@ -678,7 +677,7 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -724,7 +723,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_height = m_height; bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; - bi.bi_size = block_size; + bi.bi_weight = block_weight; bi.bi_diff = cumulative_difficulty; bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; @@ -747,7 +746,9 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - m_cum_size += block_size; + // we use weight as a proxy for size, since we don't have size but weight is >= size + // and often actually equal + m_cum_size += block_weight; m_cum_count++; } @@ -1200,8 +1201,12 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) throw DB_ERROR("Database could not be opened"); } - if (tools::is_hdd(filename.c_str())) - MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + boost::optional<bool> is_hdd_result = tools::is_hdd(filename.c_str()); + if (is_hdd_result) + { + if (is_hdd_result.value()) + MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + } m_folder = filename; @@ -1413,6 +1418,9 @@ void BlockchainLMDB::sync() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (is_read_only()) + return; + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part // MDB_NOMETASYNC. Force flush to be synchronous. if (auto result = mdb_env_sync(m_env, true)) @@ -1968,14 +1976,25 @@ std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std MDB_val v; + uint64_t prev_height = heights[0]; for (uint64_t height: heights) { - MDB_val_set(v, height); - result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (height == prev_height + 1) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + } if (result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); const mdb_block_info *bi = (const mdb_block_info *)v.mv_data; res.push_back(bi->bi_cum_rct); + prev_height = height; } TXN_POSTFIX_RDONLY(); @@ -1997,7 +2016,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2015,7 +2034,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; - size_t ret = bi->bi_size; + size_t ret = bi->bi_weight; TXN_POSTFIX_RDONLY(); return ret; } @@ -2451,55 +2470,6 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -// This is a lot harder now that we've removed the output_keys index -output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - check_open(); - TXN_PREFIX_RDONLY(); - RCURSOR(output_txs); - RCURSOR(tx_indices); - - output_data_t od; - MDB_val_set(v, global_index); - auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - outtx *ot = (outtx *)v.mv_data; - - MDB_val_set(val_h, ot->tx_hash); - get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); - if (get_result) - throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(ot->tx_hash) + ": ", get_result).c_str())); - - txindex *tip = (txindex *)val_h.mv_data; - MDB_val_set(val_tx_id, tip->data.tx_id); - MDB_val result; - get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); - - blobdata bd; - bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); - - transaction tx; - if (!parse_and_validate_tx_base_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - - const tx_out tx_output = tx.vout[ot->local_index]; - od.unlock_time = tip->data.unlock_time; - od.height = tip->data.block_id; - od.pubkey = boost::get<txout_to_key>(tx_output.target).key; - - TXN_POSTFIX_RDONLY(); - return od; -} - output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2982,10 +2952,10 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("BlockchainLMDB::" << __func__); if ((batch_transactions) && (m_batch_transactions)) { - LOG_PRINT_L0("WARNING: batch transaction mode already enabled, but asked to enable batch mode"); + MINFO("batch transaction mode already enabled, but asked to enable batch mode"); } m_batch_transactions = batch_transactions; - LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); + MINFO("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } // return true if we started the txn, false if already started @@ -3125,7 +3095,7 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3144,7 +3114,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -3733,9 +3703,9 @@ void BlockchainLMDB::migrate_0_1() if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); if (v.mv_size == sizeof(uint32_t)) - bi.bi_size = *(uint32_t *)v.mv_data; + bi.bi_weight = *(uint32_t *)v.mv_data; else - bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + bi.bi_weight = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); @@ -4294,7 +4264,7 @@ void BlockchainLMDB::migrate_2_3() bi.bi_height = bi_old->bi_height; bi.bi_timestamp = bi_old->bi_timestamp; bi.bi_coins = bi_old->bi_coins; - bi.bi_size = bi_old->bi_size; + bi.bi_weight = bi_old->bi_weight; bi.bi_diff = bi_old->bi_diff; bi.bi_hash = bi_old->bi_hash; if (bi_old->bi_height >= distribution.size()) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 8b214d2df..e1f748ed8 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -205,7 +205,7 @@ public: virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -243,7 +243,6 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -274,7 +273,7 @@ public: virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -318,7 +317,7 @@ private: uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const; virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -450,7 +449,7 @@ private: #endif #endif - constexpr static float RESIZE_PERCENT = 0.8f; + constexpr static float RESIZE_PERCENT = 0.9f; }; } // namespace cryptonote diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 338ec3e4b..37bca671f 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -28,7 +28,9 @@ set(blocksdat "") if(PER_BLOCK_CHECKPOINT) - if(APPLE) + if(APPLE AND DEPENDS) + add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} --target=x86_64-apple-darwin11 -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) + elseif(APPLE AND NOT DEPENDS) add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) else() add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) @@ -91,6 +93,28 @@ monero_private_headers(blockchain_usage +set(blockchain_ancestry_sources + blockchain_ancestry.cpp + ) + +set(blockchain_ancestry_private_headers) + +monero_private_headers(blockchain_ancestry + ${blockchain_ancestry_private_headers}) + + + +set(blockchain_depth_sources + blockchain_depth.cpp + ) + +set(blockchain_depth_private_headers) + +monero_private_headers(blockchain_depth + ${blockchain_depth_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -183,3 +207,45 @@ set_property(TARGET blockchain_usage OUTPUT_NAME "monero-blockchain-usage") install(TARGETS blockchain_usage DESTINATION bin) +monero_add_executable(blockchain_ancestry + ${blockchain_ancestry_sources} + ${blockchain_ancestry_private_headers}) + +target_link_libraries(blockchain_ancestry + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_ancestry + PROPERTY + OUTPUT_NAME "monero-blockchain-ancestry") +install(TARGETS blockchain_ancestry DESTINATION bin) + +monero_add_executable(blockchain_depth + ${blockchain_depth_sources} + ${blockchain_depth_private_headers}) + +target_link_libraries(blockchain_depth + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_depth + PROPERTY + OUTPUT_NAME "monero-blockchain-depth") +install(TARGETS blockchain_depth DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp new file mode 100644 index 000000000..2f0bbffd6 --- /dev/null +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -0,0 +1,772 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <unordered_map> +#include <unordered_set> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/archive/portable_binary_oarchive.hpp> +#include "common/unordered_containers_boost_serialization.h" +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +struct ancestor +{ + uint64_t amount; + uint64_t offset; + + bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & amount; + a & offset; + } +}; +BOOST_CLASS_VERSION(ancestor, 0) + +namespace std +{ + template<> struct hash<ancestor> + { + size_t operator()(const ancestor &a) const + { + return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't + } + }; +} + +struct tx_data_t +{ + std::vector<std::pair<uint64_t, std::vector<uint64_t>>> vin; + std::vector<crypto::public_key> vout; + bool coinbase; + + tx_data_t(): coinbase(false) {} + tx_data_t(const cryptonote::transaction &tx) + { + coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!coinbase) + { + vin.reserve(tx.vin.size()); + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets))); + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vin type"); + } + } + } + vout.reserve(tx.vout.size()); + for (size_t out = 0; out < tx.vout.size(); ++out) + { + if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx.vout[out].target); + vout.push_back(txout.key); + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vout type"); + } + } + } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & coinbase; + a & vin; + a & vout; + } +}; + +struct ancestry_state_t +{ + uint64_t height; + std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry; + std::unordered_map<ancestor, crypto::hash> output_cache; + std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; + std::vector<cryptonote::block> block_cache; + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & height; + a & ancestry; + a & output_cache; + if (ver < 1) + { + std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache; + a & old_tx_cache; + for (const auto i: old_tx_cache) + tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); + } + else + { + a & tx_cache; + } + if (ver < 2) + { + std::unordered_map<uint64_t, cryptonote::block> old_block_cache; + a & old_block_cache; + block_cache.resize(old_block_cache.size()); + for (const auto i: old_block_cache) + block_cache[i.first] = i.second; + } + else + { + a & block_cache; + } + } +}; +BOOST_CLASS_VERSION(ancestry_state_t, 2) + +static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset) +{ + std::pair<std::unordered_map<ancestor, unsigned int>::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); + if (!p.second) + { + ++p.first->second; + } +} + +static size_t get_full_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + size_t count = 0; + for (const auto &i: ancestry) + count += i.second; + return count; +} + +static size_t get_deduplicated_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + return ancestry.size(); +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const std::unordered_set<ancestor> &ancestors) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors)); + if (!p.second) + { + for (const auto &e: ancestors) + p.first->second.insert(e); + } +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>())); + p.first->second.insert(new_ancestor); +} + +static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid) +{ + std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid); + if (i == ancestry.end()) + { + //MERROR("txid ancestry not found: " << txid); + //throw std::runtime_error("txid ancestry not found"); + return std::unordered_set<ancestor>(); + } + return i->second; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_cache_outputs); + command_line::add_arg(desc_cmd_sett, arg_cache_txes); + command_line::add_arg(desc_cmd_sett, arg_cache_blocks); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_sett, arg_show_cache_stats); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_all = command_line::get_arg(vm, arg_all); + bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); + + if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + { + std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + + // forward method + if (opt_all) + { + uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; + ancestry_state_t state; + + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try + { + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); + } + state_data_in.close(); + } + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + MINFO("Starting from height " << state.height); + const uint64_t db_height = db->height(); + state.block_cache.reserve(db_height); + for (uint64_t h = state.height; h < db_height; ++h) + { + size_t block_ancestry_size = 0; + const crypto::hash block_hash = db->get_block_hash_from_height(h); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + ++total_blocks; + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(h + 1); + state.block_cache[h] = b; + } + std::vector<crypto::hash> txids; + txids.reserve(1 + b.tx_hashes.size()); + if (opt_include_coinbase) + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + for (const auto &h: b.tx_hashes) + txids.push_back(h); + for (const crypto::hash &txid: txids) + { + printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); + fflush(stdout); + ::tx_data_t tx_data; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + } + if (tx_data.coinbase) + { + add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>()); + } + else + { + for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) + { + if (1) + { + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + cryptonote::block b; + ++total_blocks; + if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[od.height]; + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + cryptonote::blobdata bd = db->get_block_blob(block_hash); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(od.height + 1); + state.block_cache[od.height] = b; + } + } + // find the tx which created this output + bool found = false; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + ++total_outputs; + if (i != state.output_cache.end()) + { + ++cached_outputs; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); + found = true; + } + else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + ::tx_data_t tx_data2; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data2 = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + tx_data2 = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); + } + for (size_t out = 0; out < tx_data2.vout.size(); ++out) + { + if (tx_data2.vout[out] == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + break; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + } + } + const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); + block_ancestry_size += ancestry_size; + MINFO(txid << ": " << ancestry_size); + } + if (!txids.empty()) + { + std::string stats_msg; + if (opt_show_cache_stats) + stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) + + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " + + std::to_string(cached_outputs*100./total_outputs); + MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); + } + state.height = h; + if (stop_requested) + break; + } + + LOG_PRINT_L0("Saving state data to " << state_file_path); + std::ofstream state_data_out; + state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (!state_data_out.fail()) + { + try + { + boost::archive::portable_binary_oarchive a(state_data_out); + a << state; + } + catch (const std::exception &e) + { + MERROR("Failed to save state data to " << state_file_path); + } + state_data_out.close(); + } + + goto done; + } + + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + for (const crypto::hash &start_txid: start_txids) + { + LOG_PRINT_L0("Checking ancestry for txid " << start_txid); + + std::unordered_map<ancestor, unsigned int> ancestry; + + std::list<crypto::hash> txids; + txids.push_back(start_txid); + while (!txids.empty()) + { + const crypto::hash txid = txids.front(); + txids.pop_front(); + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (coinbase) + continue; + + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + add_ancestor(ancestry, amount, offset); + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + + MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry)); + for (const auto &i: ancestry) + { + MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second); + } + } + +done: + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 52d2938cd..1c6e54d10 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -50,64 +50,86 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; +static const char zerokey[8] = {0}; +static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; + +static uint64_t records_per_sync = 200; +static uint64_t db_flags = 0; +static MDB_dbi dbi_relative_rings; +static MDB_dbi dbi_outputs; +static MDB_dbi dbi_processed_txidx; +static MDB_dbi dbi_spent; +static MDB_dbi dbi_ring_instances; +static MDB_dbi dbi_stats; +static MDB_env *env = NULL; + struct output_data { uint64_t amount; - uint64_t index; - output_data(): amount(0), index(0) {} - output_data(uint64_t a, uint64_t i): amount(a), index(i) {} - bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) - { - a & amount; - a & index; - } + uint64_t offset; + output_data(): amount(0), offset(0) {} + output_data(uint64_t a, uint64_t i): amount(a), offset(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.offset == offset; } }; -BOOST_CLASS_VERSION(output_data, 0) -namespace std +// +// relative_rings: key_image -> vector<uint64_t> +// outputs: 128 bits -> set of key images +// processed_txidx: string -> uint64_t +// spent: amount -> offset +// ring_instances: vector<uint64_t> -> uint64_t +// stats: string -> arbitrary +// + +static bool parse_db_sync_mode(std::string db_sync_mode) { - template<> struct hash<output_data> + std::vector<std::string> options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) { - size_t operator()(const output_data &od) const + if(options[0] == "safe") { - const uint64_t data[2] = {od.amount, od.index}; - crypto::hash h; - crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + safemode = true; + db_flags = DBF_SAFE; } - }; - template<> struct hash<std::vector<uint64_t>> - { - size_t operator()(const std::vector<uint64_t> &v) const + else if(options[0] == "fast") { - crypto::hash h; - crypto::cn_fast_hash(v.data(), v.size() * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + db_flags = DBF_FAST; } - }; -} - -struct blackball_state_t -{ - std::unordered_map<crypto::key_image, std::vector<uint64_t>> relative_rings; - std::unordered_map<output_data, std::unordered_set<crypto::key_image>> outputs; - std::unordered_map<std::string, uint64_t> processed_heights; - std::unordered_set<output_data> spent; - std::unordered_map<std::vector<uint64_t>, size_t> ring_instances; + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + db_flags = DEFAULT_FLAGS; + } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + if(options.size() >= 2 && !safemode) { - a & relative_rings; - a & outputs; - a & processed_heights; - a & spent; - if (ver < 1) - return; - a & ring_instances; + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr == '\0') + records_per_sync = bps; } -}; -BOOST_CLASS_VERSION(blackball_state_t, 1) + + return true; +} static std::string get_default_db_path() { @@ -118,7 +140,195 @@ static std::string get_default_db_path() return dir.string(); } -static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, const std::function<bool(const cryptonote::transaction_prefix&)> &f) +static std::string get_cache_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + const uint32_t *va = (const uint32_t*) a->mv_data; + const uint32_t *vb = (const uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} + +static int compare_double64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + if (va == vb) + { + const uint64_t va = ((const uint64_t*) a->mv_data)[1]; + const uint64_t vb = ((const uint64_t*) b->mv_data)[1]; + return va < vb ? -1 : va > vb; + } + return va < vb ? -1 : va > vb; +} + +static int resize_env(const char *db_path) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + size_t needed = 1000ul * 1024 * 1024; // at least 1000 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static void init(std::string cache_filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + MINFO("Creating blackball cache in " << cache_filename); + + tools::create_directories_if_necessary(cache_filename); + + int flags = 0; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + dbr = mdb_env_create(&env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 6); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_cache_filename(cache_filename); + dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "relative_rings", MDB_CREATE | MDB_INTEGERKEY, &dbi_relative_rings); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_relative_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, "outputs", MDB_CREATE | MDB_INTEGERKEY, &dbi_outputs); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_outputs, compare_double64); + + dbr = mdb_dbi_open(txn, "processed_txidx", MDB_CREATE, &dbi_processed_txidx); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_spent, compare_uint64); + + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +static void close() +{ + if (env) + { + mdb_dbi_close(env, dbi_relative_rings); + mdb_dbi_close(env, dbi_outputs); + mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_spent); + mdb_dbi_close(env, dbi_ring_instances); + mdb_dbi_close(env, dbi_stats); + mdb_env_close(env); + env = NULL; + } +} + +static std::string compress_ring(const std::vector<uint64_t> &ring, std::string s = "") +{ + const size_t sz = s.size(); + s.resize(s.size() + 12 * ring.size()); + char *ptr = (char*)s.data() + sz; + for (uint64_t out: ring) + tools::write_varint(ptr, out); + if (ptr > s.data() + sz + 12 * ring.size()) + throw std::runtime_error("varint output overflow"); + s.resize(ptr - s.data()); + return s; +} + +static std::string compress_ring(uint64_t amount, const std::vector<uint64_t> &ring) +{ + char s[12], *ptr = s; + tools::write_varint(ptr, amount); + if (ptr > s + sizeof(s)) + throw std::runtime_error("varint output overflow"); + return compress_ring(ring, std::string(s, ptr-s)); +} + +static std::vector<uint64_t> decompress_ring(const std::string &s) +{ + std::vector<uint64_t> ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + +static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, uint64_t &n_txes, const std::function<bool(const cryptonote::transaction_prefix&)> &f) { MDB_env *env; MDB_dbi dbi; @@ -126,6 +336,8 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id MDB_cursor *cur; int dbr; bool tx_active = false; + MDB_val k; + MDB_val v; dbr = mdb_env_create(&env); if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); @@ -136,7 +348,7 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open rings database file '" + actual_filename + "': " + std::string(mdb_strerror(dbr))); - dbr = mdb_txn_begin(env, NULL, 0, &txn); + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; @@ -147,9 +359,11 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn, dbi, &cur); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn, dbi, &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; - MDB_val k; - MDB_val v; bool fret = true; k.mv_size = sizeof(uint64_t); @@ -194,6 +408,77 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id return fret; } +static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) +{ + MDB_env *env[2]; + MDB_dbi dbi[2]; + MDB_txn *txn[2]; + MDB_cursor *cur[2]; + int dbr; + bool tx_active[2] = { false, false }; + uint64_t n_txes[2]; + MDB_val k; + MDB_val v[2]; + + epee::misc_utils::auto_scope_leave_caller txn_dtor[2]; + for (int i = 0; i < 2; ++i) + { + dbr = mdb_env_create(&env[i]); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env[i], 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = i ? second_filename : first_filename; + dbr = mdb_env_open(env[i], actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env[i], NULL, MDB_RDONLY, &txn[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + txn_dtor[i] = epee::misc_utils::create_scope_leave_handler([&, i](){if (tx_active[i]) mdb_txn_abort(txn[i]);}); + tx_active[i] = true; + + dbr = mdb_dbi_open(txn[i], "txs_pruned", MDB_INTEGERKEY, &dbi[i]); + if (dbr) + dbr = mdb_dbi_open(txn[i], "txs", MDB_INTEGERKEY, &dbi[i]); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn[i], dbi[i], &cur[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn[i], dbi[i], &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes[i] = stat.ms_entries; + } + + if (n_txes[0] == 0 || n_txes[1] == 0) + throw std::runtime_error("No transaction in the database"); + uint64_t lo = 0, hi = std::min(n_txes[0], n_txes[1]) - 1; + while (lo <= hi) + { + uint64_t mid = (lo + hi) / 2; + + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)∣ + dbr = mdb_cursor_get(cur[0], &k, &v[0], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cur[1], &k, &v[1], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + if (v[0].mv_size == v[1].mv_size && !memcmp(v[0].mv_data, v[1].mv_data, v[0].mv_size)) + lo = mid + 1; + else + hi = mid - 1; + } + + for (int i = 0; i < 2; ++i) + { + mdb_cursor_close(cur[i]); + mdb_txn_commit(txn[i]); + tx_active[i] = false; + mdb_dbi_close(env[i], dbi[i]); + mdb_env_close(env[i]); + } + return hi; +} + static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) { std::vector<uint64_t> c; @@ -212,6 +497,503 @@ static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) return c; } +static uint64_t get_num_spent_outputs() +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0, tmp; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + dbr = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT_NODUP; + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first/next spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &tmp); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + count += tmp; + } + + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return count; +} + +static void add_spent_output(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_put(cur, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_KEYEXIST, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); +} + +static bool is_output_spent(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); + bool spent = dbr == 0; + return spent; +} + +static std::vector<output_data> get_spent_outputs(MDB_txn *txn) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &count); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + } + std::vector<output_data> outs; + outs.reserve(count); + while (1) + { + outs.push_back({*(const uint64_t*)k.mv_data, *(const uint64_t*)v.mv_data}); + dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT); + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get next spent output: " + std::string(mdb_strerror(dbr))); + } + mdb_cursor_close(cur); + return outs; +} + +static uint64_t get_processed_txidx(const std::string &name) +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + uint64_t height = 0; + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + dbr = mdb_get(txn, dbi_processed_txidx, &k, &v); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get processed height: " + std::string(mdb_strerror(dbr))); + height = *(const uint64_t*)v.mv_data; + } + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return height; +} + +static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t height) +{ + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + v.mv_data = (void*)&height; + v.mv_size = sizeof(height); + int dbr = mdb_put(txn, dbi_processed_txidx, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set processed height: " + std::string(mdb_strerror(dbr))); +} + +static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector<uint64_t> &ring) +{ + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + int dbr = mdb_get(txn, dbi_relative_rings, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get relative ring: " + std::string(mdb_strerror(dbr))); + ring = decompress_ring(std::string((const char*)v.mv_data, v.mv_size)); + return true; +} + +static void set_relative_ring(MDB_txn *txn, const crypto::key_image &ki, const std::vector<uint64_t> &ring) +{ + const std::string sring = compress_ring(ring); + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + v.mv_data = (void*)sring.c_str(); + v.mv_size = sring.size(); + int dbr = mdb_put(txn, dbi_relative_rings, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set relative ring: " + std::string(mdb_strerror(dbr))); +} + +static std::string keep_under_511(const std::string &s) +{ + if (s.size() <= 511) + return s; + crypto::hash hash; + crypto::cn_fast_hash(s.data(), s.size(), hash); + return std::string((const char*)&hash, 32); +} + +static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + if (dbr == MDB_NOTFOUND) + return 0; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + return *(const uint64_t*)v.mv_data; +} + +static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + uint64_t instances = get_ring_instances(txn, amount, ring); + if (ring.size() > 11) + return instances; + + uint64_t extra = 0; + std::vector<uint64_t> subset; + subset.reserve(ring.size()); + for (uint64_t mask = 1; mask < (1u << ring.size()) - 1; ++mask) + { + subset.resize(0); + for (size_t i = 0; i < ring.size(); ++i) + if ((mask >> i) & 1) + subset.push_back(ring[i]); + extra += get_ring_instances(txn, amount, subset); + } + return instances + extra; +} + +static uint64_t inc_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + + uint64_t count; + if (dbr == MDB_NOTFOUND) + count = 1; + else + count = 1 + *(const uint64_t*)v.mv_data; + + v.mv_data = &count; + v.mv_size = sizeof(count); + dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set ring instances: " + std::string(mdb_strerror(dbr))); + + return count; +} + +static std::vector<crypto::key_image> get_key_images(MDB_txn *txn, const output_data &od) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return {}; + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + std::vector<crypto::key_image> key_images; + key_images.reserve(v.mv_size / 32); + const crypto::key_image *ki = (const crypto::key_image*)v.mv_data; + for (size_t n = 0; n < v.mv_size / 32; ++n) + key_images.push_back(*ki++); + return key_images; +} + +static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key_image &ki) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output"); + std::string data; + if (!dbr) + { + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + data = std::string((const char*)v.mv_data, v.mv_size); + } + data += std::string((const char*)&ki, sizeof(ki)); + + v.mv_data = (void*)data.data(); + v.mv_size = data.size(); + dbr = mdb_put(txn, dbi_outputs, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); +} + +static bool get_stat(MDB_txn *txn, const char *key, uint64_t &data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + int dbr = mdb_get(txn, dbi_stats, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get stat record"); + CHECK_AND_ASSERT_THROW_MES(v.mv_size == sizeof(uint64_t), "Unexpected record size"); + data = *(const uint64_t*)v.mv_data; + return true; +} + +static void set_stat(MDB_txn *txn, const char *key, uint64_t data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + v.mv_data = (void*)&data; + v.mv_size = sizeof(uint64_t); + int dbr = mdb_put(txn, dbi_stats, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set stat record"); +} + +static void inc_stat(MDB_txn *txn, const char *key) +{ + uint64_t data; + if (!get_stat(txn, key, data)) + data = 0; + ++data; + set_stat(txn, key, data); +} + +static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi) +{ + tools::create_directories_if_necessary(filename); + + int flags = MDB_RDONLY; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + int dbr = mdb_env_create(env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(*env, 1); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + MINFO("Opening monero blockchain at " << actual_filename); + dbr = mdb_env_open(*env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(*env, NULL, MDB_RDONLY, txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(*txn, "output_amounts", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, dbi); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(*txn, *dbi, compare_uint64); + + dbr = mdb_cursor_open(*txn, *dbi, cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); +} + +static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi) +{ + mdb_txn_abort(txn); + mdb_cursor_close(cur); + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct) +{ + uint64_t amount = 0; + MDB_val k = { sizeof(amount), (void*)&amount }, v; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + rct = 0; + } + else + { + if (dbr) throw std::runtime_error("Record 0 not found: " + std::string(mdb_strerror(dbr))); + mdb_size_t count = 0; + dbr = mdb_cursor_count(cur, &count); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + rct = count; + } + MDB_stat s; + dbr = mdb_stat(txn, dbi, &s); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + if (s.ms_entries < rct) throw std::runtime_error("Inconsistent records: " + std::string(mdb_strerror(dbr))); + pre_rct = s.ms_entries - rct; +} + +static crypto::hash get_genesis_block_hash(const std::string &filename) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 1); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "block_info", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi); + mdb_set_dupsort(txn, dbi, compare_uint64); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + uint64_t zero = 0; + MDB_val k = { sizeof(uint64_t), (void*)&zero}, v; + dbr = mdb_get(txn, dbi, &k, &v); + if (dbr) throw std::runtime_error("Failed to retrieve genesis block: " + std::string(mdb_strerror(dbr))); + crypto::hash genesis_block_hash = *(const crypto::hash*)(((const uint64_t*)v.mv_data) + 5); + mdb_dbi_close(env, dbi); + mdb_txn_abort(txn); + mdb_env_close(env); + tx_active = false; + return genesis_block_hash; +} + +static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const std::string &filename) +{ + std::vector<std::pair<uint64_t, uint64_t>> outputs; + uint64_t amount = std::numeric_limits<uint64_t>::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + if (!fgets(s, sizeof(s), f)) + { + MERROR("Error reading from " << filename << ": " << strerror(errno)); + break; + } + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair<uint64_t, uint64_t> output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets-- > 0) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +static bool export_spent_outputs(MDB_cursor *cur, const std::string &filename) +{ + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to open " << filename << ": " << strerror(errno)); + return false; + } + + uint64_t pending_amount = std::numeric_limits<uint64_t>::max(); + std::vector<uint64_t> pending_offsets; + MDB_val k, v; + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int dbr = mdb_cursor_get(cur, &k, &v, op); + if (dbr == MDB_NOTFOUND) + break; + op = MDB_NEXT; + if (dbr) + { + fclose(f); + MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr)); + return false; + } + const uint64_t amount = *(const uint64_t*)k.mv_data; + const uint64_t offset = *(const uint64_t*)v.mv_data; + if (!pending_offsets.empty() && (amount != pending_amount || pending_offsets.back()+1 != offset)) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + if (pending_amount != amount) + { + fprintf(f, "@%" PRIu64 "\n", amount); + pending_amount = amount; + } + pending_offsets.push_back(offset); + } + if (!pending_offsets.empty()) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%zu\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + fclose(f); + return true; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -231,31 +1013,37 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor<std::string, false, true, 2> arg_blackball_db_dir = { + const command_line::arg_descriptor<std::string> arg_blackball_db_dir = { "blackball-db-dir", "Specify blackball database directory", get_default_db_path(), - {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val)->std::string { - if (testnet_stagenet[0]) - return (boost::filesystem::path(val) / "testnet").string(); - else if (testnet_stagenet[1]) - return (boost::filesystem::path(val) / "stagenet").string(); - return val; - } }; const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::string> arg_database = { "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<bool> arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; + const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output)", false}; const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor<std::string> arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; + const command_line::arg_descriptor<std::string> arg_export = {"export", "Filename to export the backball list to"}; + const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_check_subsets); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); + command_line::add_arg(desc_cmd_sett, arg_export); + command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -291,11 +1079,14 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Starting..."); - bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); + std::string opt_export = command_line::get_arg(vm, arg_export); + std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); + std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -304,116 +1095,99 @@ int main(int argc, char* argv[]) return 1; } - // If we wanted to use the memory pool, we would set up a fake_core. + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + if (!parse_db_sync_mode(db_sync_mode)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } - // Use Blockchain instead of lower-level BlockchainDB for two reasons: - // 1. Blockchain has the init() method for easy setup - // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() - // - // cannot match blockchain_storage setup above with just one line, - // e.g. - // Blockchain* core_storage = new Blockchain(NULL); - // because unlike blockchain_storage constructor, which takes a pointer to - // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. - LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs); if (inputs.empty()) { LOG_PRINT_L0("No inputs given"); return 1; } - std::vector<std::unique_ptr<Blockchain>> core_storage(inputs.size()); - Blockchain *blockchain = NULL; - tx_memory_pool m_mempool(*blockchain); - for (size_t n = 0; n < inputs.size(); ++n) - { - core_storage[n].reset(new Blockchain(m_mempool)); - BlockchainDB* db = new_db(db_type); - if (db == NULL) - { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); - } - LOG_PRINT_L0("database: " << db_type); + const std::string cache_dir = (output_file_path / "blackball-cache").string(); + init(cache_dir); - std::string filename = inputs[n]; - while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) - filename.pop_back(); - LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + LOG_PRINT_L0("Scanning for blackballable outputs..."); - try - { - db->open(filename, DBF_RDONLY); - } - catch (const std::exception& e) - { - LOG_PRINT_L0("Error opening database: " << e.what()); - return 1; - } - r = core_storage[n]->init(db, net_type); + size_t done = 0; - CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); - LOG_PRINT_L0("Source blockchain storage initialized OK"); - } + const uint64_t start_blackballed_outputs = get_num_spent_outputs(); - boost::filesystem::path direc(output_file_path.string()); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - { - MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); - return 1; - } - } - else - { - if (!boost::filesystem::create_directories(direc)) - { - MERROR("Failed to create directory: " << output_file_path.string()); - return 1; - } - } + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); - LOG_PRINT_L0("Scanning for blackballable outputs..."); + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); - size_t done = 0; - blackball_state_t state; - std::unordered_set<output_data> newly_spent; - const std::string state_file_path = (boost::filesystem::path(output_file_path) / "blackball-state.bin").string(); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + + // open first db + MDB_env *env0; + MDB_txn *txn0; + MDB_dbi dbi0; + MDB_cursor *cur0; + open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) + if (!extra_spent_outputs.empty()) { - try + MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + for (const std::pair<uint64_t, uint64_t> &output: extra_spent_outputs) { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; + if (!is_output_spent(cur, output_data(output.first, output.second))) + { + blackballs.push_back(output); + add_spent_output(cur, output_data(output.first, output.second)); + inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra"); + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = blackball_state_t(); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_in.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } - uint64_t start_blackballed_outputs = state.spent.size(); - - cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); - tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); for (size_t n = 0; n < inputs.size(); ++n) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); - uint64_t start_idx = 0; - auto it = state.processed_heights.find(canonical); - if (it != state.processed_heights.end()) - start_idx = it->second; + uint64_t start_idx = get_processed_txidx(canonical); + if (n > 0 && start_idx == 0) + { + start_idx = find_first_diverging_transaction(inputs[0], inputs[n]); + LOG_PRINT_L0("First diverging transaction at " << start_idx); + } LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); - for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + size_t records = 0; + const std::string filename = inputs[n]; + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + uint64_t n_txes; + for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -425,39 +1199,65 @@ int main(int argc, char* argv[]) const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); if (n == 0) for (uint64_t out: absolute) - state.outputs[output_data(txin.amount, out)].insert(txin.k_image); + add_key_image(txn, output_data(txin.amount, out), txin.k_image); + std::vector<uint64_t> relative_ring; std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - state.ring_instances[new_ring] += 1; - if (ring_size == 1) + const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); + if (n == 0 && ring_size == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); - MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[0])); - state.spent.insert(output_data(txin.amount, absolute[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a 1-ring"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[0])); + inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } - else if (state.ring_instances[new_ring] == new_ring.size()) + else if (n == 0 && instances == new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[o])); - state.spent.insert(output_data(txin.amount, absolute[o])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[o])); + inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } - else if (state.relative_rings.find(txin.k_image) != state.relative_rings.end()) + else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) { - MINFO("Key image " << txin.k_image << " already seen: rings " << - boost::join(state.relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + for (size_t o = 0; o < new_ring.size(); ++o) + { + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[o])); + inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); + } + } + else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) + { + MDEBUG("Key image " << txin.k_image << " already seen: rings " << + boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); - if (state.relative_rings[txin.k_image] != txin.key_offsets) + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (relative_ring != txin.key_offsets) { - MINFO("Rings are different"); - const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[txin.k_image]); + MDEBUG("Rings are different"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring); const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); std::vector<uint64_t> common; for (uint64_t out: r0) @@ -468,18 +1268,24 @@ int main(int argc, char* argv[]) if (common.empty()) { MERROR("Rings for the same key image are disjoint"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } else if (common.size() == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); - MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, common[0])); - state.spent.insert(output_data(txin.amount, common[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in rings with a single common element"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, common[0])); + inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else { - MINFO("The intersection has more than one element, it's still ok"); + MDEBUG("The intersection has more than one element, it's still ok"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &out: r0) if (std::find(common.begin(), common.end(), out) != common.end()) new_ring.push_back(out); @@ -487,68 +1293,172 @@ int main(int argc, char* argv[]) } } } - state.relative_rings[txin.k_image] = new_ring; + if (n == 0) + set_relative_ring(txn, txin.k_image, new_ring); + } + set_processed_txidx(txn, canonical, start_idx+1); + + ++records; + if (records >= records_per_sync) + { + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + records = 0; + } + + if (stop_requested) + { + MINFO("Stopping scan..."); + return false; } return true; }); - state.processed_heights[canonical] = start_idx; + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx); + if (stop_requested) + break; } - while (!newly_spent.empty()) + std::vector<output_data> work_spent; + + if (stop_requested) + goto skip_secondary_passes; + + if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs) + { + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + work_spent = get_spent_outputs(txn); + mdb_txn_abort(txn); + } + + while (!work_spent.empty()) { - LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); - std::unordered_set<output_data> work_spent = std::move(newly_spent); - newly_spent.clear(); + LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs"); + + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); - for (const output_data &od: work_spent) + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + std::vector<output_data> scan_spent = std::move(work_spent); + work_spent.clear(); + for (const output_data &od: scan_spent) { - for (const crypto::key_image &ki: state.outputs[od]) + std::vector<crypto::key_image> key_images = get_key_images(txn, od); + for (const crypto::key_image &ki: key_images) { - std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[ki]); + std::vector<uint64_t> relative_ring; + CHECK_AND_ASSERT_THROW_MES(get_relative_ring(txn, ki, relative_ring), "Relative ring not found"); + std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_ring); size_t known = 0; uint64_t last_unknown = 0; for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (state.spent.find(new_od) != state.spent.end()) + if (is_output_spent(cur, new_od)) ++known; else last_unknown = out; } if (known == absolute.size() - 1) { - const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); - MINFO("Blackballing output " << pkey << ", due to being used in a " << - absolute.size() << "-ring where all other outputs are known to be spent"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(od.amount, last_unknown)); - state.spent.insert(output_data(od.amount, last_unknown)); + const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + } + blackballs.push_back(output); + add_spent_output(cur, output_data(od.amount, last_unknown)); + work_spent.push_back(output_data(od.amount, last_unknown)); + inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); } } - } - } - LOG_PRINT_L0("Saving state data to " << state_file_path); - std::ofstream state_data_out; - state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); - if (!state_data_out.fail()) - { - try - { - boost::archive::portable_binary_oarchive a(state_data_out); - a << state; + if (stop_requested) + { + MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully."); + return 0; + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to save state data to " << state_file_path); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_out.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + } + +skip_secondary_passes: + uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs; + LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs() << " total outputs blackballed"); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + uint64_t pre_rct = 0, rct = 0; + get_num_outputs(txn0, cur0, dbi0, pre_rct, rct); + MINFO("Total pre-rct outputs: " << pre_rct); + MINFO("Total rct outputs: " << rct); + static const struct { const char *key; uint64_t base; } stat_keys[] = { + { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, + { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, + { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, + { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, + { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, + { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, + }; + for (const auto &key: stat_keys) + { + uint64_t data; + if (!get_stat(txn, key.key, data)) + data = 0; + float percent = key.base ? 100.0f * data / key.base : 0.0f; + MINFO(key.key << ": " << data << " (" << percent << "%)"); + } + mdb_txn_abort(txn); + + if (!opt_export.empty()) + { + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + export_spent_outputs(cur, opt_export); + mdb_cursor_close(cur); + mdb_txn_abort(txn); } - uint64_t diff = state.spent.size() - start_blackballed_outputs; - LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << state.spent.size() << " total outputs blackballed"); LOG_PRINT_L0("Blockchain blackball data exported OK"); + close_db(env0, txn0, cur0, dbi0); + close(); return 0; - CATCH_ENTRY("Export error", 1); + CATCH_ENTRY("Error", 1); } diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp new file mode 100644 index 000000000..dd2387e5b --- /dev/null +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -0,0 +1,351 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get min depth for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get min depth for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + + if (!opt_txid_string.empty() && opt_height) + { + std::cerr << "txid and height cannot be given at the same time" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + if (opt_include_coinbase) + start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + std::vector<uint64_t> depths; + for (const crypto::hash &start_txid: start_txids) + { + uint64_t depth = 0; + bool coinbase = false; + + LOG_PRINT_L0("Checking depth for txid " << start_txid); + std::vector<crypto::hash> txids(1, start_txid); + while (!coinbase) + { + LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth); + std::vector<crypto::hash> new_txids; + for (const crypto::hash &txid: txids) + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_gen)) + { + MDEBUG(txid << " is a coinbase transaction"); + coinbase = true; + goto done; + } + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + if (!coinbase) + { + std::swap(txids, new_txids); + ++depth; + } + } +done: + LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); + depths.push_back(depth); + } + + uint64_t cumulative_depth = 0; + for (uint64_t depth: depths) + cumulative_depth += depth; + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size()); + LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << epee::misc_utils::median(depths)); + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 7291dbd68..9ec768d26 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -474,17 +474,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path txs.push_back(tx); } - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; - block_size = bp.block_size; + block_weight = bp.block_weight; cumulative_difficulty = bp.cumulative_difficulty; coins_generated = bp.coins_generated; try { - core.get_blockchain_storage().get_db().add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index ba2697226..beaad2abc 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -236,11 +236,11 @@ void BootstrapFile::write_block(block& block) bool include_extra_block_data = true; if (include_extra_block_data) { - size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); + size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height); difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); - bp.block_size = block_size; + bp.block_weight = block_weight; bp.cumulative_difficulty = cumulative_difficulty; bp.coins_generated = coins_generated; } diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 8755d3fe3..278a7b40f 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -70,14 +70,14 @@ namespace cryptonote { cryptonote::block block; std::vector<transaction> txs; - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; BEGIN_SERIALIZE() FIELD(block) FIELD(txs) - VARINT_FIELD(block_size) + VARINT_FIELD(block_weight) VARINT_FIELD(cumulative_difficulty) VARINT_FIELD(coins_generated) END_SERIALIZE() diff --git a/src/checkpoints/CMakeLists.txt b/src/checkpoints/CMakeLists.txt index 02bb2891a..715006522 100644 --- a/src/checkpoints/CMakeLists.txt +++ b/src/checkpoints/CMakeLists.txt @@ -27,9 +27,13 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. if(APPLE) - find_library(IOKIT_LIBRARY IOKit) - mark_as_advanced(IOKIT_LIBRARY) - list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + if(DEPENDS) + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework ApplicationServices -framework AppKit -framework IOKit") + else() + find_library(IOKIT_LIBRARY IOKit) + mark_as_advanced(IOKIT_LIBRARY) + list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + endif() endif() set(checkpoints_sources diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index f0df05b0d..c6bac2199 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -38,7 +38,8 @@ set(common_sources password.cpp perf_timer.cpp threadpool.cpp - updates.cpp) + updates.cpp + aligned.c) if (STACK_TRACE) list(APPEND common_sources stack_trace.cpp) @@ -67,7 +68,8 @@ set(common_private_headers perf_timer.h stack_trace.h threadpool.h - updates.h) + updates.h + aligned.h) monero_private_headers(common ${common_private_headers}) diff --git a/src/common/aligned.c b/src/common/aligned.c new file mode 100644 index 000000000..763dfd0e7 --- /dev/null +++ b/src/common/aligned.c @@ -0,0 +1,139 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <stdint.h> +#include <string.h> +#include "aligned.h" + +static inline int is_power_of_2(size_t n) { return n && (n & (n-1)) == 0; } + +#define MAGIC 0xaa0817161500ff81 +#define MAGIC_FREED 0xaa0817161500ff82 + +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +typedef struct +{ + uint64_t magic; + void *raw; + size_t bytes; + size_t align; +} control; + +void *aligned_malloc(size_t bytes, size_t align) +{ + void *raw, *ptr; + control *ctrl; + + if (!is_power_of_2(align)) + return NULL; + if (bytes > (size_t)-1 - align) + return NULL; + if (bytes + align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + align); + if (!raw) + return NULL; + ptr = (void*)(((uintptr_t)raw + align + sizeof(control) - 1) & ~(align-1)); + ctrl = ((control*)ptr) - 1; + ctrl->magic = MAGIC; + ctrl->raw = raw; + ctrl->bytes = bytes; + ctrl->align = align; + return ptr; +} + +void *aligned_realloc(void *ptr, size_t bytes, size_t align) +{ + void *raw, *ptr2; + control *ctrl, *ctrl2; + + if (!ptr) + return aligned_malloc(bytes, align); + if (!bytes) + { + aligned_free(ptr); + return NULL; + } + if (!is_power_of_2(align)) + return NULL; + + ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + if (ctrl->align != align) + return NULL; + if (ctrl->bytes >= bytes) + return ptr; + + if (ctrl->bytes > (size_t)-1 - ctrl->align) + return NULL; + if (ctrl->bytes + ctrl->align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + ctrl->align); + if (!raw) + return NULL; + ptr2 = (void*)(((uintptr_t)raw + ctrl->align + sizeof(control) - 1) & ~(ctrl->align-1)); + memcpy(ptr2, ptr, ctrl->bytes); + ctrl2 = ((control*)ptr2) - 1; + ctrl2->magic = MAGIC; + ctrl2->raw = raw; + ctrl2->bytes = bytes; + ctrl2->align = ctrl->align; + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); + return ptr2; +} + +void aligned_free(void *ptr) +{ + if (!ptr) + return; + control *ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); +} diff --git a/src/common/aligned.h b/src/common/aligned.h new file mode 100644 index 000000000..fed3ccb36 --- /dev/null +++ b/src/common/aligned.h @@ -0,0 +1,41 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +void *aligned_malloc(size_t bytes, size_t align); +void *aligned_realloc(void *ptr, size_t bytes, size_t align); +void aligned_free(void *ptr); + +#ifdef __cplusplus +} +#endif diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 33f60bc3c..3f2bde620 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -97,11 +97,16 @@ get_builtin_cert(void) */ /** return the built in root DS trust anchor */ -static const char* +static const char* const* get_builtin_ds(void) { - return -". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n"; + static const char * const ds[] = + { + ". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n", + ". IN DS 20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", + NULL + }; + return ds; } /************************************************************ @@ -240,7 +245,12 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - ub_ctx_add_ta(m_data->m_ub_context, string_copy(::get_builtin_ds())); + const char * const *ds = ::get_builtin_ds(); + while (*ds) + { + MINFO("adding trust anchor: " << *ds); + ub_ctx_add_ta(m_data->m_ub_context, string_copy(*ds++)); + } } DNSResolver::~DNSResolver() diff --git a/src/common/json_util.h b/src/common/json_util.h index 661022a6f..c320c3956 100644 --- a/src/common/json_util.h +++ b/src/common/json_util.h @@ -29,14 +29,14 @@ #pragma once #define GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, name, type, jtype, mandatory, def) \ - type field_##name = def; \ + type field_##name = static_cast<type>(def); \ bool field_##name##_found = false; \ (void)field_##name##_found; \ do if (json.HasMember(#name)) \ { \ if (json[#name].Is##jtype()) \ { \ - field_##name = json[#name].Get##jtype(); \ + field_##name = static_cast<type>(json[#name].Get##jtype()); \ field_##name##_found = true; \ } \ else \ diff --git a/src/common/password.cpp b/src/common/password.cpp index 3ce2ba42a..5671c4a4e 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -54,7 +54,7 @@ namespace return 0 != _isatty(_fileno(stdin)); } - bool read_from_tty(epee::wipeable_string& pass) + bool read_from_tty(epee::wipeable_string& pass, bool hide_input) { static constexpr const char BACKSPACE = 8; @@ -62,7 +62,7 @@ namespace DWORD mode_old; ::GetConsoleMode(h_cin, &mode_old); - DWORD mode_new = mode_old & ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT); + DWORD mode_new = mode_old & ~((hide_input ? ENABLE_ECHO_INPUT : 0) | ENABLE_LINE_INPUT); ::SetConsoleMode(h_cin, mode_new); bool r = true; @@ -107,14 +107,14 @@ namespace return 0 != isatty(fileno(stdin)); } - int getch() noexcept + int getch(bool hide_input) noexcept { struct termios tty_old; tcgetattr(STDIN_FILENO, &tty_old); struct termios tty_new; tty_new = tty_old; - tty_new.c_lflag &= ~(ICANON | ECHO); + tty_new.c_lflag &= ~(ICANON | (hide_input ? ECHO : 0)); tcsetattr(STDIN_FILENO, TCSANOW, &tty_new); int ch = getchar(); @@ -124,14 +124,14 @@ namespace return ch; } - bool read_from_tty(epee::wipeable_string& aPass) + bool read_from_tty(epee::wipeable_string& aPass, bool hide_input) { static constexpr const char BACKSPACE = 127; aPass.reserve(tools::password_container::max_password_size); while (aPass.size() < tools::password_container::max_password_size) { - int ch = getch(); + int ch = getch(hide_input); if (EOF == ch || ch == EOT) { return false; @@ -159,18 +159,18 @@ namespace #endif // end !WIN32 - bool read_from_tty(const bool verify, const char *message, epee::wipeable_string& pass1, epee::wipeable_string& pass2) + bool read_from_tty(const bool verify, const char *message, bool hide_input, epee::wipeable_string& pass1, epee::wipeable_string& pass2) { while (true) { if (message) std::cout << message <<": " << std::flush; - if (!read_from_tty(pass1)) + if (!read_from_tty(pass1, hide_input)) return false; if (verify) { std::cout << "Confirm password: "; - if (!read_from_tty(pass2)) + if (!read_from_tty(pass2, hide_input)) return false; if(pass1!=pass2) { @@ -229,12 +229,12 @@ namespace tools std::atomic<bool> password_container::is_prompting(false); - boost::optional<password_container> password_container::prompt(const bool verify, const char *message) + boost::optional<password_container> password_container::prompt(const bool verify, const char *message, bool hide_input) { is_prompting = true; password_container pass1{}; password_container pass2{}; - if (is_cin_tty() ? read_from_tty(verify, message, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) + if (is_cin_tty() ? read_from_tty(verify, message, hide_input, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) { is_prompting = false; return {std::move(pass1)}; diff --git a/src/common/password.h b/src/common/password.h index 61937b93a..529881e40 100644 --- a/src/common/password.h +++ b/src/common/password.h @@ -49,7 +49,7 @@ namespace tools password_container(std::string&& password) noexcept; //! \return A password from stdin TTY prompt or `std::cin` pipe. - static boost::optional<password_container> prompt(bool verify, const char *mesage = "Password"); + static boost::optional<password_container> prompt(bool verify, const char *mesage = "Password", bool hide_input = true); static std::atomic<bool> is_prompting; password_container(const password_container&) = delete; diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 16abdfd99..6910ebdd4 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -33,7 +33,7 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "perf" -namespace +namespace tools { uint64_t get_tick_count() { @@ -83,7 +83,7 @@ namespace tools el::Level performance_timer_log_level = el::Level::Debug; -static __thread std::vector<PerformanceTimer*> *performance_timers = NULL; +static __thread std::vector<LoggingPerformanceTimer*> *performance_timers = NULL; void set_performance_timer_log_level(el::Level level) { @@ -96,17 +96,24 @@ void set_performance_timer_log_level(el::Level level) performance_timer_log_level = level; } -PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Level l): name(s), unit(unit), level(l), started(false), paused(false) +PerformanceTimer::PerformanceTimer(bool paused): started(true), paused(paused) +{ + if (paused) + ticks = 0; + else + ticks = get_tick_count(); +} + +LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l): PerformanceTimer(), name(s), unit(unit), level(l) { - ticks = get_tick_count(); if (!performance_timers) { MLOG(level, "PERF ----------"); - performance_timers = new std::vector<PerformanceTimer*>(); + performance_timers = new std::vector<LoggingPerformanceTimer*>(); } else { - PerformanceTimer *pt = performance_timers->back(); + LoggingPerformanceTimer *pt = performance_timers->back(); if (!pt->started && !pt->paused) { size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; @@ -119,9 +126,14 @@ PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Leve PerformanceTimer::~PerformanceTimer() { - performance_timers->pop_back(); if (!paused) ticks = get_tick_count() - ticks; +} + +LoggingPerformanceTimer::~LoggingPerformanceTimer() +{ + pause(); + performance_timers->pop_back(); char s[12]; snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index 0e910caf9..675d6234d 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -43,30 +43,46 @@ class PerformanceTimer; extern el::Level performance_timer_log_level; +uint64_t get_tick_count(); +uint64_t get_ticks_per_ns(); +uint64_t ticks_to_ns(uint64_t ticks); + class PerformanceTimer { public: - PerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + PerformanceTimer(bool paused = false); ~PerformanceTimer(); void pause(); void resume(); + uint64_t value() const { return ticks; } +void set(uint64_t v){ticks=v;} + +protected: + uint64_t ticks; + bool started; + bool paused; +}; + +class LoggingPerformanceTimer: public PerformanceTimer +{ +public: + LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + ~LoggingPerformanceTimer(); + private: std::string name; uint64_t unit; el::Level level; - uint64_t ticks; - bool started; - bool paused; }; void set_performance_timer_log_level(el::Level level); -#define PERF_TIMER_UNIT(name, unit) tools::PerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) -#define PERF_TIMER_UNIT_L(name, unit, l) tools::PerformanceTimer pt_##name(#name, unit, l) +#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) +#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, unit, l) #define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000) #define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000, l) -#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::PerformanceTimer> pt_##name(new tools::PerformanceTimer(#name, unit, el::Level::Info)) +#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> pt_##name(new tools::LoggingPerformanceTimer(#name, unit, el::Level::Info)) #define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000) #define PERF_TIMER_STOP(name) do { pt_##name.reset(NULL); } while(0) #define PERF_TIMER_PAUSE(name) pt_##name->pause() diff --git a/src/common/stack_trace.cpp b/src/common/stack_trace.cpp index d6dc4d7cc..141621427 100644 --- a/src/common/stack_trace.cpp +++ b/src/common/stack_trace.cpp @@ -49,7 +49,16 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "stacktrace" -#define ST_LOG(x) CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,MONERO_DEFAULT_LOG_CATEGORY) << x +#define ST_LOG(x) \ + do { \ + auto elpp = ELPP; \ + if (elpp) { \ + CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,MONERO_DEFAULT_LOG_CATEGORY) << x; \ + } \ + else { \ + std::cout << x << std::endl; \ + } \ + } while(0) // from https://stackoverflow.com/questions/11665829/how-can-i-print-stack-trace-for-caught-exceptions-in-c-code-injection-in-c diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 9eb402e0b..9f12f8dbc 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -69,12 +69,12 @@ namespace tools continue; bool alnum = true; - for (auto c: hash) + for (auto c: fields[3]) if (!isalnum(c)) alnum = false; - if (hash.size() != 64 && !alnum) + if (fields[3].size() != 64 && !alnum) { - MWARNING("Invalid hash: " << hash); + MWARNING("Invalid hash: " << fields[3]); continue; } diff --git a/src/common/util.cpp b/src/common/util.cpp index 7d9d7b408..c56c77505 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -37,7 +37,7 @@ #ifdef __GLIBC__ #include <sys/types.h> #include <sys/stat.h> -#include <ustat.h> +#include <sys/resource.h> #include <unistd.h> #include <dirent.h> #include <string.h> @@ -45,6 +45,13 @@ #include <string> #endif +//tools::is_hdd +#ifdef __GLIBC__ + #include <sstream> + #include <sys/sysmacros.h> + #include <fstream> +#endif + #include "unbound.h" #include "include_base_utils.h" @@ -682,6 +689,21 @@ std::string get_nix_version_display_string() static void setup_crash_dump() {} #endif + bool disable_core_dumps() + { +#ifdef __GLIBC__ + // disable core dumps in release mode + struct rlimit rlimit; + rlimit.rlim_cur = rlimit.rlim_max = 0; + if (setrlimit(RLIMIT_CORE, &rlimit)) + { + MWARNING("Failed to disable core dumps"); + return false; + } +#endif + return true; + } + bool on_startup() { mlog_configure("", true); @@ -717,62 +739,41 @@ std::string get_nix_version_display_string() #endif } - bool is_hdd(const char *path) + boost::optional<bool> is_hdd(const char *file_path) { #ifdef __GLIBC__ - std::string device = ""; - struct stat st, dst; - if (stat(path, &st) < 0) - return 0; - - DIR *dir = opendir("/dev/block"); - if (!dir) - return 0; - struct dirent *de; - while ((de = readdir(dir))) - { - if (strcmp(de->d_name, ".") && strcmp(de->d_name, "..")) + struct stat st; + std::string prefix; + if(stat(file_path, &st) == 0) + { + std::ostringstream s; + s << "/sys/dev/block/" << major(st.st_dev) << ":" << minor(st.st_dev); + prefix = s.str(); + } + else + { + return boost::none; + } + std::string attr_path = prefix + "/queue/rotational"; + std::ifstream f(attr_path, std::ios_base::in); + if(not f.is_open()) + { + attr_path = prefix + "/../queue/rotational"; + f.open(attr_path, std::ios_base::in); + if(not f.is_open()) { - std::string dev_path = std::string("/dev/block/") + de->d_name; - char resolved[PATH_MAX]; - if (realpath(dev_path.c_str(), resolved) && !strncmp(resolved, "/dev/", 5)) - { - if (stat(resolved, &dst) == 0) - { - if (dst.st_rdev == st.st_dev) - { - // take out trailing digits (eg, sda1 -> sda) - char *ptr = resolved; - while (*ptr) - ++ptr; - while (ptr > resolved && isdigit(*--ptr)) - *ptr = 0; - device = resolved + 5; - break; - } - } - } + return boost::none; } } - closedir(dir); - - if (device.empty()) - return 0; - - std::string sys_path = "/sys/block/" + device + "/queue/rotational"; - FILE *f = fopen(sys_path.c_str(), "r"); - if (!f) - return false; - char s[8]; - char *ptr = fgets(s, sizeof(s), f); - fclose(f); - if (!ptr) - return 0; - s[sizeof(s) - 1] = 0; - int n = atoi(s); // returns 0 on parse error - return n == 1; + unsigned short val = 0xdead; + f >> val; + if(not f.fail()) + { + return (val == 1); + } + return boost::none; #else - return 0; + return boost::none; #endif } @@ -919,4 +920,23 @@ std::string get_nix_version_display_string() return {}; } } + + std::string glob_to_regex(const std::string &val) + { + std::string newval; + + bool escape = false; + for (char c: val) + { + if (c == '*') + newval += escape ? "*" : ".*"; + else if (c == '?') + newval += escape ? "?" : "."; + else if (c == '\\') + newval += '\\', escape = !escape; + else + newval += c; + } + return newval; + } } diff --git a/src/common/util.h b/src/common/util.h index a57a85fee..0e0b50520 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -149,6 +149,8 @@ namespace tools bool sanitize_locale(); + bool disable_core_dumps(); + bool on_startup(); /*! \brief Defines a signal handler for win32 and *nix @@ -228,7 +230,9 @@ namespace tools bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash); bool sha256sum(const std::string &filename, crypto::hash &hash); - bool is_hdd(const char *path); + boost::optional<bool> is_hdd(const char *path); boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str); + + std::string glob_to_regex(const std::string &val); } diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 71dcedcab..0c635e7cb 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -78,6 +78,7 @@ target_link_libraries(cncrypto PUBLIC epee ${Boost_SYSTEM_LIBRARY} + ${SODIUM_LIBRARY} PRIVATE ${EXTRA_LIBRARIES}) diff --git a/src/crypto/chacha.h b/src/crypto/chacha.h index 2b3ed8043..6e85ad0e9 100644 --- a/src/crypto/chacha.h +++ b/src/crypto/chacha.h @@ -40,6 +40,7 @@ #include <memory.h> #include "memwipe.h" +#include "mlocker.h" #include "hash.h" namespace crypto { @@ -50,7 +51,7 @@ namespace crypto { #if defined(__cplusplus) } - using chacha_key = tools::scrubbed_arr<uint8_t, CHACHA_KEY_SIZE>; + using chacha_key = epee::mlocked<tools::scrubbed_arr<uint8_t, CHACHA_KEY_SIZE>>; #pragma pack(push, 1) // MS VC 2012 doesn't interpret `class chacha_iv` as POD in spite of [9.0.10], so it is a struct @@ -69,22 +70,26 @@ namespace crypto { chacha20(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher); } - inline void generate_chacha_key(const void *data, size_t size, chacha_key& key) { + inline void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr<char, HASH_SIZE> pwd_hash; + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + for (uint64_t n = 1; n < kdf_rounds; ++n) + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } - inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key) { + inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr<char, HASH_SIZE> pwd_hash; + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 1/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + for (uint64_t n = 1; n < kdf_rounds; ++n) + crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } - inline void generate_chacha_key(std::string password, chacha_key& key) { - return generate_chacha_key(password.data(), password.size(), key); + inline void generate_chacha_key(std::string password, chacha_key& key, uint64_t kdf_rounds) { + return generate_chacha_key(password.data(), password.size(), key, kdf_rounds); } } diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 4ff4310de..1f77513ca 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -871,3 +871,9 @@ const fe fe_fffb2 = {8166131, -6741800, -17040804, 3154616, 21461005, 1466302, - const fe fe_fffb3 = {-13620103, 14639558, 4532995, 7679154, 16815101, -15883539, -22863840, -14813421, 13716513, -6477756}; /* sqrt(-sqrt(-1) * A * (A + 2)) */ const fe fe_fffb4 = {-21786234, -12173074, 21573800, 4524538, -4645904, 16204591, 8012863, -8444712, 3212926, 6885324}; /* sqrt(sqrt(-1) * A * (A + 2)) */ const ge_p3 ge_p3_identity = { {0}, {1, 0}, {1, 0}, {0} }; +const ge_p3 ge_p3_H = { + {7329926, -15101362, 31411471, 7614783, 27996851, -3197071, -11157635, -6878293, 466949, -7986503}, + {5858699, 5096796, 21321203, -7536921, -5553480, -11439507, -5627669, 15045946, 19977121, 5275251}, + {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {23443568, -5110398, -8776029, -4345135, 6889568, -14710814, 7474843, 3279062, 14550766, -7453428} +}; diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 45d412ac6..09296d6f9 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -3707,9 +3707,8 @@ void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, s[31] = s11 >> 17; } -/* Assumes that a != INT64_MIN */ static int64_t signum(int64_t a) { - return (a >> 63) - ((-a) >> 63); + return a > 0 ? 1 : a < 0 ? -1 : 0; } int sc_check(const unsigned char *s) { @@ -3730,3 +3729,16 @@ int sc_isnonzero(const unsigned char *s) { s[18] | s[19] | s[20] | s[21] | s[22] | s[23] | s[24] | s[25] | s[26] | s[27] | s[28] | s[29] | s[30] | s[31]) - 1) >> 8) + 1; } + +int ge_p3_is_point_at_infinity(const ge_p3 *p) { + // X = 0 and Y == Z + int n; + for (n = 0; n < 10; ++n) + { + if (p->X[n] | p->T[n]) + return 0; + if (p->Y[n] != p->Z[n]) + return 0; + } + return 1; +} diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index dc3c60794..2910dafd4 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -140,6 +140,7 @@ extern const fe fe_fffb2; extern const fe fe_fffb3; extern const fe fe_fffb4; extern const ge_p3 ge_p3_identity; +extern const ge_p3 ge_p3_H; void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); void sc_0(unsigned char *); void sc_reduce32(unsigned char *); @@ -158,3 +159,5 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); + +int ge_p3_is_point_at_infinity(const ge_p3 *p); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 0fd6d9363..ad7721cf0 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -70,6 +70,9 @@ namespace crypto { #include "random.h" } + const crypto::public_key null_pkey = crypto::public_key{}; + const crypto::secret_key null_skey = crypto::secret_key{}; + static inline unsigned char *operator &(ec_point &point) { return &reinterpret_cast<unsigned char &>(point); } @@ -93,12 +96,32 @@ namespace crypto { generate_random_bytes_not_thread_safe(N, bytes); } + static inline bool less32(const unsigned char *k0, const unsigned char *k1) + { + for (int n = 31; n >= 0; --n) + { + if (k0[n] < k1[n]) + return true; + if (k0[n] > k1[n]) + return false; + } + return false; + } + + void random32_unbiased(unsigned char *bytes) + { + // l = 2^252 + 27742317777372353535851937790883648493. + // it fits 15 in 32 bytes + static const unsigned char limit[32] = { 0xe3, 0x6a, 0x67, 0x72, 0x8b, 0xce, 0x13, 0x29, 0x8f, 0x30, 0x82, 0x8c, 0x0b, 0xa4, 0x10, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0 }; + do + { + generate_random_bytes_thread_safe(32, bytes); + } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time + sc_reduce32(bytes); + } /* generate a random 32-byte (256-bit) integer and copy it to res */ static inline void random_scalar(ec_scalar &res) { - unsigned char tmp[64]; - generate_random_bytes_thread_safe(64, tmp); - sc_reduce(tmp); - memcpy(&res, tmp, 32); + random32_unbiased((unsigned char*)res.data); } void hash_to_scalar(const void *data, size_t length, ec_scalar &res) { @@ -251,11 +274,18 @@ namespace crypto { #endif buf.h = prefix_hash; buf.key = pub; + try_again: random_scalar(k); + if (((const uint32_t*)(&k))[7] == 0) // we don't want tiny numbers here + goto try_again; ge_scalarmult_base(&tmp3, &k); ge_p3_tobytes(&buf.comm, &tmp3); hash_to_scalar(&buf, sizeof(s_comm), sig.c); + if (!sc_isnonzero((const unsigned char*)sig.c.data)) + goto try_again; sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k); + if (!sc_isnonzero((const unsigned char*)sig.r.data)) + goto try_again; } bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) { @@ -269,11 +299,14 @@ namespace crypto { if (ge_frombytes_vartime(&tmp3, &pub) != 0) { return false; } - if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0) { + if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0 || !sc_isnonzero(&sig.c)) { return false; } ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r); ge_tobytes(&buf.comm, &tmp2); + static const ec_point infinity = {{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + if (memcmp(&buf.comm, &infinity, 32) == 0) + return false; hash_to_scalar(&buf, sizeof(s_comm), c); sc_sub(&c, &c, &sig.c); return sc_isnonzero(&c) == 0; diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 9ea0f2ec0..33cc0a25a 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -34,7 +34,6 @@ #include <iostream> #include <boost/thread/mutex.hpp> #include <boost/thread/lock_guard.hpp> -#include <boost/utility/value_init.hpp> #include <boost/optional.hpp> #include <type_traits> #include <vector> @@ -42,6 +41,7 @@ #include "common/pod-class.h" #include "common/util.h" #include "memwipe.h" +#include "mlocker.h" #include "generic-ops.h" #include "hex.h" #include "span.h" @@ -66,7 +66,7 @@ namespace crypto { friend class crypto_ops; }; - using secret_key = tools::scrubbed<ec_scalar>; + using secret_key = epee::mlocked<tools::scrubbed<ec_scalar>>; POD_CLASS public_keyV { std::vector<public_key> keys; @@ -99,6 +99,7 @@ namespace crypto { #pragma pack(pop) void hash_to_scalar(const void *data, size_t length, ec_scalar &res); + void random32_unbiased(unsigned char *bytes); static_assert(sizeof(ec_point) == 32 && sizeof(ec_scalar) == 32 && sizeof(public_key) == 32 && sizeof(secret_key) == 32 && @@ -277,11 +278,11 @@ namespace crypto { epee::to_hex::formatted(o, epee::as_byte_span(v)); return o; } - const static crypto::public_key null_pkey = boost::value_initialized<crypto::public_key>(); - const static crypto::secret_key null_skey = boost::value_initialized<crypto::secret_key>(); + const extern crypto::public_key null_pkey; + const extern crypto::secret_key null_skey; } CRYPTO_MAKE_HASHABLE(public_key) -CRYPTO_MAKE_HASHABLE(secret_key) +CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key) CRYPTO_MAKE_HASHABLE(key_image) CRYPTO_MAKE_COMPARABLE(signature) diff --git a/src/crypto/crypto_ops_builder/ietf.txt b/src/crypto/crypto_ops_builder/ietf.txt index 0736f71ec..609f5e75a 100644 --- a/src/crypto/crypto_ops_builder/ietf.txt +++ b/src/crypto/crypto_ops_builder/ietf.txt @@ -1,1402 +1,4 @@ - +https://tools.ietf.org/id/draft-josefsson-eddsa-ed25519-02.txt -[Docs] [txt|pdf] [Tracker] [Email] [Diff1] [Diff2] [Nits] - -Versions: 00 01 02 - -Network Working Group S. Josefsson -Internet-Draft SJD AB -Intended status: Informational N. Moeller -Expires: August 26, 2015 - February 22, 2015 - - - EdDSA and Ed25519 - - draft-josefsson-eddsa-ed25519-02 - - -Abstract - - The elliptic curve signature scheme EdDSA and one instance of it - called Ed25519 is described. An example implementation and test - vectors are provided. - -Status of This Memo - - This Internet-Draft is submitted in full conformance with the - provisions of BCP 78 and BCP 79. - - Internet-Drafts are working documents of the Internet Engineering - Task Force (IETF). Note that other groups may also distribute - working documents as Internet-Drafts. The list of current Internet- - Drafts is at http://datatracker.ietf.org/drafts/current/. - - Internet-Drafts are draft documents valid for a maximum of six months - and may be updated, replaced, or obsoleted by other documents at any - time. It is inappropriate to use Internet-Drafts as reference - material or to cite them other than as "work in progress." - - This Internet-Draft will expire on August 26, 2015. - -Copyright Notice - - Copyright (c) 2015 IETF Trust and the persons identified as the - document authors. All rights reserved. - - This document is subject to BCP 78 and the IETF Trust's Legal - Provisions Relating to IETF Documents - (http://trustee.ietf.org/license-info) in effect on the date of - publication of this document. Please review these documents - carefully, as they describe your rights and restrictions with respect - to this document. Code Components extracted from this document must - include Simplified BSD License text as described in Section 4.e of - the Trust Legal Provisions and are provided without warranty as - described in the Simplified BSD License. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 1] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -Table of Contents - - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Notation . . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 3. Background . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 4. EdDSA . . . . . . . . . . . . . . . . . . . . . . . . . . . . 4 - 4.1. Encoding . . . . . . . . . . . . . . . . . . . . . . . . 4 - 4.2. Keys . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 4.3. Sign . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 4.4. Verify . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 5. Ed25519 . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 - 5.1. Modular arithmetic . . . . . . . . . . . . . . . . . . . 6 - 5.2. Encoding . . . . . . . . . . . . . . . . . . . . . . . . 6 - 5.3. Decoding . . . . . . . . . . . . . . . . . . . . . . . . 6 - 5.4. Point addition . . . . . . . . . . . . . . . . . . . . . 7 - 5.5. Key Generation . . . . . . . . . . . . . . . . . . . . . 8 - 5.6. Sign . . . . . . . . . . . . . . . . . . . . . . . . . . 8 - 5.7. Verify . . . . . . . . . . . . . . . . . . . . . . . . . 9 - 5.8. Python illustration . . . . . . . . . . . . . . . . . . . 9 - 6. Test Vectors for Ed25519 . . . . . . . . . . . . . . . . . . 14 - 7. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . 17 - 8. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 18 - 9. Security Considerations . . . . . . . . . . . . . . . . . . . 18 - 9.1. Side-channel leaks . . . . . . . . . . . . . . . . . . . 18 - 10. References . . . . . . . . . . . . . . . . . . . . . . . . . 18 - 10.1. Normative References . . . . . . . . . . . . . . . . . . 18 - 10.2. Informative References . . . . . . . . . . . . . . . . . 18 - Appendix A. Ed25519 Python Library . . . . . . . . . . . . . . . 19 - Appendix B. Library driver . . . . . . . . . . . . . . . . . . . 23 - Authors' Addresses . . . . . . . . . . . . . . . . . . . . . . . 24 - -1. Introduction - - - The Edwards-curve Digital Signature Algorithm (EdDSA) is a variant of - Schnorr's signature system with Twisted Edwards curves. EdDSA needs - to be instantiated with certain parameters and this document describe - Ed25519 - an instantiation of EdDSA in a curve over GF(2^255-19). To - facilitate adoption in the Internet community of Ed25519, this - document describe the signature scheme in an implementation-oriented - way, and we provide sample code and test vectors. - - The advantages with EdDSA and Ed25519 include: - - 1. High-performance on a variety of platforms. - - 2. Does not require the use of a unique random number for each - signature. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 2] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 3. More resilient to side-channel attacks. - - 4. Small public keys (32 bytes) and signatures (64 bytes). - - 5. The formulas are "strongly unified", i.e., they are valid for all - points on the curve, with no exceptions. This obviates the need - for EdDSA to perform expensive point validation on untrusted - public values. - - 6. Collision resilience, meaning that hash-function collisions do - not break this system. - - For further background, see the original EdDSA paper [EDDSA]. - -2. Notation - - - The following notation is used throughout the document: - - GF(p) finite field with p elements - - x^y x multiplied by itself y times - - B generator of the group or subgroup of interest - - n B B added to itself n times. - - h_i the i'th bit of h - - a || b (bit-)string a concatenated with (bit-)string b - -3. Background - - - EdDSA is defined using an elliptic curve over GF(p) of the form - - -x^2 + y^2 = 1 + d x^2 y^2 - - In general, p could be a prime power, but it is usually chosen as a - prime number. It is required that p = 1 modulo 4 (which implies that - -1 is a square modulo p) and that d is a non-square modulo p. For - Ed25519, the curve used is equivalent to Curve25519 [CURVE25519], - under a change of coordinates, which means that the difficulty of the - discrete logarithm problem is the same as for Curve25519. - - Points on this curve form a group under addition, (x3, y3) = (x1, y1) - + (x2, y2), with the formulas - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 3] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - x1 y2 + x2 y1 y1 y2 + x1 x2 - x3 = -------------------, y3 = ------------------- - 1 + d x1 x2 y1 y2 1 - d x1 x2 y1 y2 - - The neutral element in the group is (0, 1). - - Unlike manyy other curves used for cryptographic applications, these - formulas are "strongly unified": they are valid for all points on the - curve, with no exceptions. In particular, the denominators are non- - zero for all input points. - - There are more efficient formulas, which are still strongly unified, - which use homogeneous coordinates to avoid the expensive modulo p - inversions. See [Faster-ECC] and [Edwards-revisited]. - -4. EdDSA - - - EdDSA is a digital signature system with several parameters. The - generic EdDSA digital signature system is normally not implemented - directly, but instead a particular instance of EdDSA (like Ed25519) - is implemented. A precise explanation of the generic EdDSA is thus - not particulary useful for implementers, but for background and - completeness, a succint description of the generic EdDSA algorithm is - given here. - - EdDSA has seven parameters: - - 1. an integer b >= 10. - - 2. a cryptographic hash function H producing 2b-bit outputs. - - 3. a prime power p congruent to 1 modulo 4. - - 4. a (b-1)-bit encoding of elements of the finite field GF(p). - - 5. a non-square element d of GF(p) - - 6. an element B != (0,1) of the set E = { (x,y) is a member of GF(p) - x GF(p) such that -x^2 + y^2 = 1 + dx^2y^2 }. - - 7. a prime q, of size b-3 bits, such that qB = (0, 1), i.e., q is - the order of B or a multiple thereof. - -4.1. Encoding - - - An element (x,y) of E is encoded as a b-bit string called ENC(x,y) - which is the (b-1)-bit encoding of y concatenated with one bit that - is 1 if x is negative and 0 if x is not negative. Negative elements - - - -Josefsson & Moeller Expires August 26, 2015 [Page 4] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - of GF(q) are those x which the (b-1)-bit encoding of x is - lexicographically larger than the (b-1)-bit encoding of -x. - -4.2. Keys - - - An EdDSA secret key is a b-bit string k. Let the hash H(k) = (h_0, - h_1, ..., h_(2b-1)) determine an integer a which is 2^(b-2) plus the - sum of m = 2^i * h_i for all i equal or larger than 3 and equal to or - less than b-3 such that m is a member of the set { 2^(b-2), 2^(b-2) + - 8, ..., 2^(b-1) - 8 }. The EdDSA public key is ENC(A) = ENC(aB). - The bits h_b, ..., h_(2b-1) is used below during signing. - -4.3. Sign - - - The signature of a message M under a secret key k is the 2b-bit - string ENC(R) || ENC'(S), where ENC'(S) is defined as the b-bit - little-endian encoding of S. R and S are derived as follows. First - define r = H(h_b, ... h_(2b-1)), M) interpreting 2b-bit strings in - little-endian form as integers in {0, 1, ..., 2^(2b)-1}. Let R=rB - and S=(r+H(ENC(R) || ENC(A) || M)a) mod l. - -4.4. Verify - - - To verify a signature ENC(R) || ENC'(S) on a message M under a public - key ENC(A), proceed as follows. Parse the inputs so that A and R is - an element of E, and S is a member of the set {0, 1, ..., l-1 }. - Compute H' = H(ENC(R) || ENC(A) || M) and check the group equation - 8SB = 8R + 8H'A in E. Verification is rejected if parsing fails or - the group equation does not hold. - -5. Ed25519 - - - Theoretically, Ed25519 is EdDSA instantiated with b=256, H being - SHA-512 [RFC4634], p is the prime 2^255-19, the 255-bit encoding of - GF(2^255-19) being the little-endian encoding of {0, 1, ..., - 2^255-20}, q is the prime 2^252 + 0x14def9dea2f79cd65812631a5cf5d3ed, - d = -121665/121666 which is a member of GF(p), and B is the unique - point (x, 4/5) in E for which x is "positive", which with the - encoding used simply means that the least significant bit of x is 0. - The curve p, prime q, d and B follows from [I-D.irtf-cfrg-curves]. - - Written out explicitly, B is the point (15112221349535400772501151409 - 588531511454012693041857206046113283949847762202, 4631683569492647816 - 9428394003475163141307993866256225615783033603165251855960). - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 5] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -5.1. Modular arithmetic - - - For advise on how to implement arithmetic modulo p = 2^255 - 1 - efficiently and securely, see Curve25519 [CURVE25519]. For inversion - modulo p, it is recommended to use the identity x^-1 = x^(p-2) (mod - p). - - For point decoding or "decompression", square roots modulo p are - needed. They can be computed using the Tonelli-Shanks algorithm, or - the special case for p = 5 (mod 8). To find a square root of a, - first compute the candidate root x = a^((p+3)/8) (mod p). Then there - are three cases: - - x^2 = a (mod p). Then x is a square root. - - x^2 = -a (mod p). Then 2^((p-1)/4) x is a square root. - - a is not a square modulo p. - -5.2. Encoding - - - All values are coded as octet strings, and integers are coded using - little endian convention. I.e., a 32-octet string h h[0],...h[31] - represents the integer h[0] + 2^8 h[1] + ... + 2^248 h[31]. - - A curve point (x,y), with coordiantes in the range 0 <= x,y < p, is - coded as follows. First encode the y-coordinate as a little-endian - string of 32 octets. The most significant bit of the final octet is - always zero. To form the encoding of the point, copy the least - significant bit of the x-coordinate to the most significant bit of - the final octet. - -5.3. Decoding - - - Decoding a point, given as a 32-octet string, is a little more - complicated. - - 1. First interpret the string as an integer in little-endian - representation. Bit 255 of this number is the least significant - bit of the x-coordinate, and denote this value x_0. The - y-coordinate is recovered simply by clearing this bit. If the - resulting value is >= p, decoding fails. - - 2. To recover the x coordinate, the curve equation implies x^2 = - (y^2 - 1) / (d y^2 + 1) (mod p). Since d is a non-square and -1 - is a square, the numerator, (d y^2 + 1), is always invertible - modulo p. Let u = y^2 - 1 and v = d y^2 + 1. To compute the - square root of (u/v), the first step is to compute the candidate - - - -Josefsson & Moeller Expires August 26, 2015 [Page 6] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - root x = (u/v)^((p+3)/8). This can be done using the following - trick, to use a single modular powering for both the inversion of - v and the square root: - - (p+3)/8 3 (p-5)/8 - x = (u/v) = u v (u v^7) (mod p) - - 3. Again, there are three cases: - - 1. If v x^2 = u (mod p), x is a square root. - - 2. If v x^2 = -u (mod p), set x <-- x 2^((p-1)/4), which is a - square root. - - 3. Otherwise, no square root exists modulo p, and decoding - fails. - - 4. Finally, use the x_0 bit to select the right square root. If x = - 0, and x_0 = 1, decoding fails. Otherwise, if x_0 != x mod 2, - set x <-- p - x. Return the decoded point (x,y). - -5.4. Point addition - - - For point addition, the following method is recommended. A point - (x,y) is represented in extended homogeneous coordinates (X, Y, Z, - T), with x = X/Z, y = Y/Z, x y = T/Z. - - The following formulas for adding two points, (x3,y3) = - (x1,y1)+(x2,y2) are described in [Edwards-revisited], section 3.1. - They are strongly unified, i.e., they work for any pair of valid - input points. - - A = (Y1-X1)*(Y2-X2) - B = (Y1+X1)*(Y2+X2) - C = T1*2*d*T2 - D = Z1*2*Z2 - E = B-A - F = D-C - G = D+C - H = B+A - X3 = E*F - Y3 = G*H - T3 = E*H - Z3 = F*G - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 7] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -5.5. Key Generation - - - The secret is 32 octets (256 bits, corresponding to b) of - cryptographically-secure random data. See [RFC4086] for a discussion - about randomness. - - The 32-byte public key is generated by the following steps. - - 1. Hash the 32-byte secret using SHA-512, storing the digest in a - 64-octet large buffer, denoted h. Only the lower 32 bytes are - used for generating the public key. - - 2. Prune the buffer. In C terminology: - - h[0] &= ~0x07; - h[31] &= 0x7F; - h[31] |= 0x40; - - 3. Interpret the buffer as the little-endian integer, forming a - secret scalar a. Perform a known-base-point scalar - multiplication a B. - - 4. The public key A is the encoding of the point aB. First encode - the y coordinate (in the range 0 <= y < p) as a little-endian - string of 32 octets. The most significant bit of the final octet - is always zero. To form the encoding of the point aB, copy the - least significant bit of the x coordinate to the most significant - bit of the final octet. The result is the public key. - -5.6. Sign - - - The imputs to the signing procedure is the secret key, a 32-octet - string, and a message M of arbitrary size. - - 1. Hash the secret key, 32-octets, using SHA-512. Let h denote the - resulting digest. Construct the secret scalar a from the first - half of the digest, and the corresponding public key A, as - described in the previous section. Let prefix denote the second - half of the hash digest, h[32],...,h[63]. - - 2. Compute SHA-512(prefix || M), where M is the message to be - signed. Interpret the 64-octet digest as a little-endian integer - r. - - 3. Compute the point rB. For efficiency, do this by first reducing - r modulo q, the group order of B. Let the string R be the - encoding of this point. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 8] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 4. Compute SHA512(R || A || M), and interpret the 64-octet digest as - a little-endian integer k. - - 5. Compute s = (r + k a) mod q. For efficiency, again reduce k - modulo q first. - - 6. Form the signature of the concatenation of R (32 octets) and the - little-endian encoding of s (32 octets, three most significant - bits of the final octets always zero). - -5.7. Verify - - - 1. To verify a signature on a message M, first split the signature - into two 32-octet halves. Decode the first half as a point R, - and the second half as an integer s, in the range 0 <= s < q. If - the decoding fails, the signature is invalid. - - 2. Compute SHA512(R || A || M), and interpret the 64-octet digest as - a little-endian integer k. - - 3. Check the group equation 8s B = 8 R + 8k A. It's sufficient, but - not required, to instead check s B = R + k A. - -5.8. Python illustration - - - The rest of this section describes how Ed25519 can be implemented in - Python (version 3.2 or later) for illustration. See appendix A for - the complete implementation and appendix B for a test-driver to run - it through some test vectors. - - First some preliminaries that will be needed. - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 9] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - import hashlib - - def sha512(s): - return hashlib.sha512(s).digest() - - # Base field Z_p - p = 2**255 - 19 - - def modp_inv(x): - return pow(x, p-2, p) - - # Curve constant - d = -121665 * modp_inv(121666) % p - - # Group order - q = 2**252 + 27742317777372353535851937790883648493 - - def sha512_modq(s): - return int.from_bytes(sha512(s), "little") % q - - Then follows functions to perform point operations. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 10] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -# Points are represented as tuples (X, Y, Z, T) of extended coordinates, -# with x = X/Z, y = Y/Z, x*y = T/Z - -def point_add(P, Q): - A = (P[1]-P[0])*(Q[1]-Q[0]) % p - B = (P[1]+P[0])*(Q[1]+Q[0]) % p - C = 2 * P[3] * Q[3] * d % p - D = 2 * P[2] * Q[2] % p - E = B-A - F = D-C - G = D+C - H = B+A - return (E*F, G*H, F*G, E*H) - -# Computes Q = s * Q -def point_mul(s, P): - Q = (0, 1, 1, 0) # Neutral element - while s > 0: - # Is there any bit-set predicate? - if s & 1: - Q = point_add(Q, P) - P = point_add(P, P) - s >>= 1 - return Q - -def point_equal(P, Q): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - if (P[0] * Q[2] - Q[0] * P[2]) % p != 0: - return False - if (P[1] * Q[2] - Q[1] * P[2]) % p != 0: - return False - return True - - Now follows functions for point compression. - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 11] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -# Square root of -1 -modp_sqrt_m1 = pow(2, (p-1) // 4, p) - -# Compute corresponding x coordinate, with low bit corresponding to sign, -# or return None on failure -def recover_x(y, sign): - x2 = (y*y-1) * modp_inv(d*y*y+1) - if x2 == 0: - if sign: - return None - else: - return 0 - - # Compute square root of x2 - x = pow(x2, (p+3) // 8, p) - if (x*x - x2) % p != 0: - x = x * modp_sqrt_m1 % p - if (x*x - x2) % p != 0: - return None - - if (x & 1) != sign: - x = p - x - return x - -# Base point -g_y = 4 * modp_inv(5) % p -g_x = recover_x(g_y, 0) -G = (g_x, g_y, 1, g_x * g_y % p) - -def point_compress(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - -def point_decompress(s): - if len(s) != 32: - raise Exception("Invalid input length for decompression") - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - - x = recover_x(y, sign) - if x is None: - return None - else: - return (x, y, 1, x*y % p) - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 12] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - These are functions for manipulating the secret. - - def secret_expand(secret): - if len(secret) != 32: - raise Exception("Bad size of private key") - h = sha512(secret) - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= (1 << 254) - return (a, h[32:]) - - def secret_to_public(secret): - (a, dummy) = secret_expand(secret) - return point_compress(point_mul(a, G)) - - The signature function works as below. - - def sign(secret, msg): - a, prefix = secret_expand(secret) - A = point_compress(point_mul(a, G)) - r = sha512_modq(prefix + msg) - R = point_mul(r, G) - Rs = point_compress(R) - h = sha512_modq(Rs + A + msg) - s = (r + h * a) % q - return Rs + int.to_bytes(s, 32, "little") - - And finally the verification function. - - def verify(public, msg, signature): - if len(public) != 32: - raise Exception("Bad public-key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - h = sha512_modq(Rs + public + msg) - sB = point_mul(s, G) - hA = point_mul(h, A) - return point_equal(sB, point_add(R, hA)) - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 13] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -6. Test Vectors for Ed25519 - - - Below is a sequence of octets with test vectors for the the Ed25519 - signature algorithm. The octets are hex encoded and whitespace is - inserted for readability. Private keys are 64 bytes, public keys 32 - bytes, message of arbitrary length, and signatures are 64 bytes. The - test vectors are taken from [ED25519-TEST-VECTORS] (but we removed - the public key as a suffix of the secret key, and removed the message - from the signature) and [ED25519-LIBGCRYPT-TEST-VECTORS]. - - -----TEST 1 - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - d75a980182b10ab7d54bfed3c964073a - 0ee172f3daa62325af021a68f707511a - - MESSAGE (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - - -----TEST 2 - SECRET KEY: - 4ccd089b28ff96da9db6c346ec114e0f - 5b8a319f35aba624da8cf6ed4fb8a6fb - - PUBLIC KEY: - 3d4017c3e843895a92b70aa74d1b7ebc - 9c982ccf2ec4968cc0cd55f12af4660c - - MESSAGE (length 1 byte): - 72 - - SIGNATURE: - 92a009a9f0d4cab8720e820b5f642540 - a2b27b5416503f8fb3762223ebdb69da - 085ac1e43e15996e458f3613d0f11d8c - 387b2eaeb4302aeeb00d291612bb0c00 - - -----TEST 3 - SECRET KEY: - c5aa8df43f9f837bedb7442f31dcb7b1 - - - -Josefsson & Moeller Expires August 26, 2015 [Page 14] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - 66d38535076f094b85ce3a2e0b4458f7 - - PUBLIC KEY: - fc51cd8e6218a1a38da47ed00230f058 - 0816ed13ba3303ac5deb911548908025 - - MESSAGE (length 2 bytes): - af82 - - SIGNATURE: - 6291d657deec24024827e69c3abe01a3 - 0ce548a284743a445e3680d7db5ac3ac - 18ff9b538d16f290ae67f760984dc659 - 4a7c15e9716ed28dc027beceea1ec40a - - -----TEST 1024 - SECRET KEY: - f5e5767cf153319517630f226876b86c - 8160cc583bc013744c6bf255f5cc0ee5 - - PUBLIC KEY: - 278117fc144c72340f67d0f2316e8386 - ceffbf2b2428c9c51fef7c597f1d426e - - MESSAGE: - 08b8b2b733424243760fe426a4b54908 - 632110a66c2f6591eabd3345e3e4eb98 - fa6e264bf09efe12ee50f8f54e9f77b1 - e355f6c50544e23fb1433ddf73be84d8 - 79de7c0046dc4996d9e773f4bc9efe57 - 38829adb26c81b37c93a1b270b20329d - 658675fc6ea534e0810a4432826bf58c - 941efb65d57a338bbd2e26640f89ffbc - 1a858efcb8550ee3a5e1998bd177e93a - 7363c344fe6b199ee5d02e82d522c4fe - ba15452f80288a821a579116ec6dad2b - 3b310da903401aa62100ab5d1a36553e - 06203b33890cc9b832f79ef80560ccb9 - a39ce767967ed628c6ad573cb116dbef - efd75499da96bd68a8a97b928a8bbc10 - 3b6621fcde2beca1231d206be6cd9ec7 - aff6f6c94fcd7204ed3455c68c83f4a4 - 1da4af2b74ef5c53f1d8ac70bdcb7ed1 - 85ce81bd84359d44254d95629e9855a9 - 4a7c1958d1f8ada5d0532ed8a5aa3fb2 - d17ba70eb6248e594e1a2297acbbb39d - 502f1a8c6eb6f1ce22b3de1a1f40cc24 - 554119a831a9aad6079cad88425de6bd - - - -Josefsson & Moeller Expires August 26, 2015 [Page 15] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - e1a9187ebb6092cf67bf2b13fd65f270 - 88d78b7e883c8759d2c4f5c65adb7553 - 878ad575f9fad878e80a0c9ba63bcbcc - 2732e69485bbc9c90bfbd62481d9089b - eccf80cfe2df16a2cf65bd92dd597b07 - 07e0917af48bbb75fed413d238f5555a - 7a569d80c3414a8d0859dc65a46128ba - b27af87a71314f318c782b23ebfe808b - 82b0ce26401d2e22f04d83d1255dc51a - ddd3b75a2b1ae0784504df543af8969b - e3ea7082ff7fc9888c144da2af58429e - c96031dbcad3dad9af0dcbaaaf268cb8 - fcffead94f3c7ca495e056a9b47acdb7 - 51fb73e666c6c655ade8297297d07ad1 - ba5e43f1bca32301651339e22904cc8c - 42f58c30c04aafdb038dda0847dd988d - cda6f3bfd15c4b4c4525004aa06eeff8 - ca61783aacec57fb3d1f92b0fe2fd1a8 - 5f6724517b65e614ad6808d6f6ee34df - f7310fdc82aebfd904b01e1dc54b2927 - 094b2db68d6f903b68401adebf5a7e08 - d78ff4ef5d63653a65040cf9bfd4aca7 - 984a74d37145986780fc0b16ac451649 - de6188a7dbdf191f64b5fc5e2ab47b57 - f7f7276cd419c17a3ca8e1b939ae49e4 - 88acba6b965610b5480109c8b17b80e1 - b7b750dfc7598d5d5011fd2dcc5600a3 - 2ef5b52a1ecc820e308aa342721aac09 - 43bf6686b64b2579376504ccc493d97e - 6aed3fb0f9cd71a43dd497f01f17c0e2 - cb3797aa2a2f256656168e6c496afc5f - b93246f6b1116398a346f1a641f3b041 - e989f7914f90cc2c7fff357876e506b5 - 0d334ba77c225bc307ba537152f3f161 - 0e4eafe595f6d9d90d11faa933a15ef1 - 369546868a7f3a45a96768d40fd9d034 - 12c091c6315cf4fde7cb68606937380d - b2eaaa707b4c4185c32eddcdd306705e - 4dc1ffc872eeee475a64dfac86aba41c - 0618983f8741c5ef68d3a101e8a3b8ca - c60c905c15fc910840b94c00a0b9d0 - - SIGNATURE: - 0aab4c900501b3e24d7cdf4663326a3a - 87df5e4843b2cbdb67cbf6e460fec350 - aa5371b1508f9f4528ecea23c436d94b - 5e8fcd4f681e30a6ac00a9704a188a03 - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 16] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - -----TEST 1A - -----An additional test with the data from test 1 but using an - -----uncompressed public key. - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - 0455d0e09a2b9d34292297e08d60d0f6 - 20c513d47253187c24b12786bd777645 - ce1a5107f7681a02af2523a6daf372e1 - 0e3a0764c9d3fe4bd5b70ab18201985a - d7 - - MSG (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - - -----TEST 1B - -----An additional test with the data from test 1 but using an - -----compressed prefix. - SECRET KEY: - 9d61b19deffd5a60ba844af492ec2cc4 - 4449c5697b326919703bac031cae7f60 - - PUBLIC KEY: - 40d75a980182b10ab7d54bfed3c96407 - 3a0ee172f3daa62325af021a68f70751 - 1a - - MESSAGE (length 0 bytes): - - SIGNATURE: - e5564300c360ac729086e2cc806e828a - 84877f1eb8e5d974d873e06522490155 - 5fb8821590a33bacc61e39701cf9b46b - d25bf5f0595bbe24655141438e7a100b - ----- - -7. Acknowledgements - - - Feedback on this document was received from Werner Koch and Damien - Miller. - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 17] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -8. IANA Considerations - - - None. - -9. Security Considerations - - -9.1. Side-channel leaks - - - For implementations performing signatures, secrecy of the key is - fundamental. It is possible to protect against some side-channel - attacks by ensuring that the implementation executes exactly the same - sequence of instructions and performs exactly the same memory - accesses, for any value of the secret key. - - To make an implementation side-channel silent in this way, the modulo - p arithmetic must not use any data-dependent branches, e.g., related - to carry propagation. Side channel-silent point addition is - straight-forward, thanks to the unified formulas. - - Scalar multiplication, multiplying a point by an integer, needs some - additional effort to implement in a side-channel silent manner. One - simple approach is to implement a side-channel silent conditional - assignment, and use together with the binary algorithm to examine one - bit of the integer at a time. - - Note that the example implementation in this document does not - attempt to be side-channel silent. - -10. References - - -10.1. Normative References - - - [RFC4634] Eastlake, D. and T. Hansen, "US Secure Hash Algorithms - (SHA and HMAC-SHA)", RFC 4634, July 2006. - - [I-D.irtf-cfrg-curves] - Langley, A., Salz, R., and S. Turner, "Elliptic Curves for - Security", draft-irtf-cfrg-curves-01 (work in progress), - January 2015. - -10.2. Informative References - - - [RFC4086] Eastlake, D., Schiller, J., and S. Crocker, "Randomness - Requirements for Security", BCP 106, RFC 4086, June 2005. - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 18] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - [EDDSA] Bernstein, D., Duif, N., Lange, T., Schwabe, P., and B. - Yang, "High-speed high-security signatures", WWW - http://ed25519.cr.yp.to/ed25519-20110926.pdf, September - 2011. - - [Faster-ECC] - Bernstein, D. and T. Lange, "Faster addition and doubling - on elliptic curves", WWW http://eprint.iacr.org/2007/286, - July 2007. - - [Edwards-revisited] - Hisil, H., Wong, K., Carter, G., and E. Dawson, "Twisted - Edwards Curves Revisited", WWW - http://eprint.iacr.org/2008/522, December 2008. - - [CURVE25519] - Bernstein, D., "Curve25519: new Diffie-Hellman speed - records", WWW http://cr.yp.to/ecdh.html, February 2006. - - [ED25519-TEST-VECTORS] - Bernstein, D., Duif, N., Lange, T., Schwabe, P., and B. - Yang, "Ed25519 test vectors", WWW - http://ed25519.cr.yp.to/python/sign.input, July 2011. - - [ED25519-LIBGCRYPT-TEST-VECTORS] - Koch, W., "Ed25519 Libgcrypt test vectors", WWW - http://git.gnupg.org/cgi- - bin/gitweb.cgi?p=libgcrypt.git;a=blob;f=tests/t-ed25519.in - p;h=e13566f826321eece65e02c593bc7d885b3dbe23;hb=refs/ - heads/master, July 2014. - -Appendix A. Ed25519 Python Library - - - Below is an example implementation of Ed25519 written in Python, - version 3.2 or higher is required. - -# Loosely based on the public domain code at -# http://ed25519.cr.yp.to/software.html -# -# Needs python-3.2 - -import hashlib - - -def sha512(s): - return hashlib.sha512(s).digest() - -# Base field Z_p - - - -Josefsson & Moeller Expires August 26, 2015 [Page 19] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - -p = 2**255 - 19 - - -def modp_inv(x): - return pow(x, p-2, p) - -# Curve constant -d = -121665 * modp_inv(121666) % p - -# Group order -q = 2**252 + 27742317777372353535851937790883648493 - - -def sha512_modq(s): - return int.from_bytes(sha512(s), "little") % q - -# Points are represented as tuples (X, Y, Z, T) of extended coordinates, -# with x = X/Z, y = Y/Z, x*y = T/Z - - -def point_add(P, Q): - A = (P[1]-P[0])*(Q[1]-Q[0]) % p - B = (P[1]+P[0])*(Q[1]+Q[0]) % p - C = 2 * P[3] * Q[3] * d % p - D = 2 * P[2] * Q[2] % p - E = B-A - F = D-C - G = D+C - H = B+A - return (E*F, G*H, F*G, E*H) - - -# Computes Q = s * Q -def point_mul(s, P): - Q = (0, 1, 1, 0) # Neutral element - while s > 0: - # Is there any bit-set predicate? - if s & 1: - Q = point_add(Q, P) - P = point_add(P, P) - s >>= 1 - return Q - - -def point_equal(P, Q): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - if (P[0] * Q[2] - Q[0] * P[2]) % p != 0: - return False - - - -Josefsson & Moeller Expires August 26, 2015 [Page 20] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - if (P[1] * Q[2] - Q[1] * P[2]) % p != 0: - return False - return True - -# Square root of -1 -modp_sqrt_m1 = pow(2, (p-1) // 4, p) - - -# Compute corresponding x coordinate, with low bit corresponding to sign, -# or return None on failure -def recover_x(y, sign): - x2 = (y*y-1) * modp_inv(d*y*y+1) - if x2 == 0: - if sign: - return None - else: - return 0 - - # Compute square root of x2 - x = pow(x2, (p+3) // 8, p) - if (x*x - x2) % p != 0: - x = x * modp_sqrt_m1 % p - if (x*x - x2) % p != 0: - return None - - if (x & 1) != sign: - x = p - x - return x - -# Base point -g_y = 4 * modp_inv(5) % p -g_x = recover_x(g_y, 0) -G = (g_x, g_y, 1, g_x * g_y % p) - - -def point_compress(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - - -def point_decompress(s): - if len(s) != 32: - raise Exception("Invalid input length for decompression") - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - - - -Josefsson & Moeller Expires August 26, 2015 [Page 21] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - x = recover_x(y, sign) - if x is None: - return None - else: - return (x, y, 1, x*y % p) - - -def secret_expand(secret): - if len(secret) != 32: - raise Exception("Bad size of private key") - h = sha512(secret) - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= (1 << 254) - return (a, h[32:]) - - -def secret_to_public(secret): - (a, dummy) = secret_expand(secret) - return point_compress(point_mul(a, G)) - - -def sign(secret, msg): - a, prefix = secret_expand(secret) - A = point_compress(point_mul(a, G)) - r = sha512_modq(prefix + msg) - R = point_mul(r, G) - Rs = point_compress(R) - h = sha512_modq(Rs + A + msg) - s = (r + h * a) % q - return Rs + int.to_bytes(s, 32, "little") - - -def verify(public, msg, signature): - if len(public) != 32: - raise Exception("Bad public-key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - h = sha512_modq(Rs + public + msg) - sB = point_mul(s, G) - - - -Josefsson & Moeller Expires August 26, 2015 [Page 22] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - hA = point_mul(h, A) - return point_equal(sB, point_add(R, hA)) - -Appendix B. Library driver - - - Below is a command-line tool that uses the library above to perform - computations, for interactive use or for self-checking. - - import sys - import binascii - - from ed25519 import * - - def point_valid(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - assert (x*y - P[3]*zinv) % p == 0 - return (-x*x + y*y - 1 - d*x*x*y*y) % p == 0 - - assert point_valid(G) - Z = (0, 1, 1, 0) - assert point_valid(Z) - - assert point_equal(Z, point_add(Z, Z)) - assert point_equal(G, point_add(Z, G)) - assert point_equal(Z, point_mul(0, G)) - assert point_equal(G, point_mul(1, G)) - assert point_equal(point_add(G, G), point_mul(2, G)) - for i in range(0, 100): - assert point_valid(point_mul(i, G)) - assert point_equal(Z, point_mul(q, G)) - - def munge_string(s, pos, change): - return (s[:pos] + - int.to_bytes(s[pos] ^ change, 1, "little") + - s[pos+1:]) - - # Read a file in the format of - # http://ed25519.cr.yp.to/python/sign.input - lineno = 0 - while True: - line = sys.stdin.readline() - if not line: - break - lineno = lineno + 1 - print(lineno) - fields = line.split(":") - - - -Josefsson & Moeller Expires August 26, 2015 [Page 23] - - -Internet-Draft EdDSA & Ed25519 February 2015 - - - secret = (binascii.unhexlify(fields[0]))[:32] - public = binascii.unhexlify(fields[1]) - msg = binascii.unhexlify(fields[2]) - signature = binascii.unhexlify(fields[3])[:64] - - assert public == secret_to_public(secret) - assert signature == sign(secret, msg) - assert verify(public, msg, signature) - if len(msg) == 0: - bad_msg = b"x" - else: - bad_msg = munge_string(msg, len(msg) // 3, 4) - assert not verify(public, bad_msg, signature) - bad_signature = munge_string(signature, 20, 8) - assert not verify(public, msg, bad_signature) - bad_signature = munge_string(signature, 40, 16) - assert not verify(public, msg, bad_signature) - -Authors' Addresses - - Simon Josefsson - SJD AB - - Email: simon@josefsson.org - URI: http://josefsson.org/ - - - Niels Moeller - - Email: nisse@lysator.liu.se - - - - - - - - - - - - - - - - - - - - - -Josefsson & Moeller Expires August 26, 2015 [Page 24] - - - -Html markup produced by rfcmarkup 1.113, available from https://tools.ietf.org/tools/rfcmarkup/ +Note: This draft is now superseded by https://datatracker.ietf.org/doc/rfc8032/ +(review of the differences is left as an exercise for the reader) diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 62bc758c9..42b98706e 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -33,19 +33,30 @@ #include <cstddef> #include <cstring> #include <functional> +#include <sodium/crypto_verify_32.h> #define CRYPTO_MAKE_COMPARABLE(type) \ namespace crypto { \ inline bool operator==(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) == 0; \ + return !memcmp(&_v1, &_v2, sizeof(_v1)); \ } \ inline bool operator!=(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) != 0; \ + return !operator==(_v1, _v2); \ } \ } -#define CRYPTO_MAKE_HASHABLE(type) \ -CRYPTO_MAKE_COMPARABLE(type) \ +#define CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +namespace crypto { \ + inline bool operator==(const type &_v1, const type &_v2) { \ + static_assert(sizeof(_v1) == 32, "constant time comparison is only implenmted for 32 bytes"); \ + return crypto_verify_32((const unsigned char*)&_v1, (const unsigned char*)&_v2) == 0; \ + } \ + inline bool operator!=(const type &_v1, const type &_v2) { \ + return !operator==(_v1, _v2); \ + } \ +} + +#define CRYPTO_DEFINE_HASH_FUNCTIONS(type) \ namespace crypto { \ static_assert(sizeof(std::size_t) <= sizeof(type), "Size of " #type " must be at least that of size_t"); \ inline std::size_t hash_value(const type &_v) { \ @@ -60,3 +71,12 @@ namespace std { \ } \ }; \ } + +#define CRYPTO_MAKE_HASHABLE(type) \ +CRYPTO_MAKE_COMPARABLE(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + +#define CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(type) \ +CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index de8e2a5b3..8fcd2138e 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -132,3 +132,77 @@ void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md) { keccak(in, inlen, md, sizeof(state_t)); } + +#define KECCAK_FINALIZED 0x80000000 +#define KECCAK_BLOCKLEN 136 +#define KECCAK_WORDS 17 +#define KECCAK_DIGESTSIZE 32 +#define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0))) +#define KECCAK_PROCESS_BLOCK(st, block) { \ + for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \ + ((st))[i_] ^= ((block))[i_]; \ + }; \ + keccakf(st, KECCAK_ROUNDS); } + + +void keccak_init(KECCAK_CTX * ctx){ + memset(ctx, 0, sizeof(KECCAK_CTX)); +} + +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen){ + if (ctx->rest & KECCAK_FINALIZED) { + local_abort("Bad keccak use"); + } + + const size_t idx = ctx->rest; + ctx->rest = (ctx->rest + inlen) % KECCAK_BLOCKLEN; + + // fill partial block + if (idx) { + size_t left = KECCAK_BLOCKLEN - idx; + memcpy((char*)ctx->message + idx, in, (inlen < left ? inlen : left)); + if (inlen < left) return; + + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + + in += left; + inlen -= left; + } + + const bool is_aligned = IS_ALIGNED_64(in); + while (inlen >= KECCAK_BLOCKLEN) { + const uint64_t* aligned_message_block; + if (is_aligned) { + aligned_message_block = (uint64_t*)in; + } else { + memcpy(ctx->message, in, KECCAK_BLOCKLEN); + aligned_message_block = ctx->message; + } + + KECCAK_PROCESS_BLOCK(ctx->hash, aligned_message_block); + in += KECCAK_BLOCKLEN; + inlen -= KECCAK_BLOCKLEN; + } + if (inlen) { + memcpy(ctx->message, in, inlen); + } +} + +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){ + if (!(ctx->rest & KECCAK_FINALIZED)) + { + // clear the rest of the data queue + memset((char*)ctx->message + ctx->rest, 0, KECCAK_BLOCKLEN - ctx->rest); + ((char*)ctx->message)[ctx->rest] |= 0x01; + ((char*)ctx->message)[KECCAK_BLOCKLEN - 1] |= 0x80; + + // process final block + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + ctx->rest = KECCAK_FINALIZED; // mark context as finalized + } + + static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, ""); + if (md) { + memcpy(md, ctx->hash, KECCAK_DIGESTSIZE); + } +} diff --git a/src/crypto/keccak.h b/src/crypto/keccak.h index fb9d8bd04..9123c7a3b 100644 --- a/src/crypto/keccak.h +++ b/src/crypto/keccak.h @@ -15,6 +15,17 @@ #define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) #endif +// SHA3 Algorithm context. +typedef struct KECCAK_CTX +{ + // 1600 bits algorithm hashing state + uint64_t hash[25]; + // 1088-bit buffer for leftovers, block size = 136 B for 256-bit keccak + uint64_t message[17]; + // count of bytes in the message[] buffer + size_t rest; +} KECCAK_CTX; + // compute a keccak hash (md) of given byte length from "in" void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen); @@ -23,4 +34,7 @@ void keccakf(uint64_t st[25], int norounds); void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md); +void keccak_init(KECCAK_CTX * ctx); +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen); +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md); #endif diff --git a/src/crypto/oaes_lib.c b/src/crypto/oaes_lib.c index 9e31ebf46..210f5d43a 100644 --- a/src/crypto/oaes_lib.c +++ b/src/crypto/oaes_lib.c @@ -33,14 +33,15 @@ #include <stdlib.h> #include <stdio.h> -// OS X, FreeBSD, and OpenBSD don't need malloc.h +// OS X, FreeBSD, OpenBSD and NetBSD don't need malloc.h #if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) \ - && !defined(__DragonFly__) + && !defined(__DragonFly__) && !defined(__NetBSD__) #include <malloc.h> #endif -// ANDROID, FreeBSD, and OpenBSD also don't need timeb.h -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) +// ANDROID, FreeBSD, OpenBSD and NetBSD also don't need timeb.h +#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) \ + && !defined(__NetBSD__) #include <sys/timeb.h> #else #include <sys/time.h> @@ -473,7 +474,7 @@ OAES_RET oaes_sprintf( #ifdef OAES_HAVE_ISAAC static void oaes_get_seed( char buf[RANDSIZ + 1] ) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; @@ -505,7 +506,7 @@ static void oaes_get_seed( char buf[RANDSIZ + 1] ) #else static uint32_t oaes_get_seed(void) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 9d4fc0dfa..914ba6dc0 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -38,6 +38,7 @@ #include "common/int-util.h" #include "hash-ops.h" #include "oaes_lib.h" +#include "variant2_int_sqrt.h" #define MEMORY (1 << 21) // 2MB scratchpad #define ITER (1 << 20) @@ -50,7 +51,7 @@ extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expa extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); #define VARIANT1_1(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ const uint8_t tmp = ((const uint8_t*)(p))[11]; \ static const uint32_t table = 0x75310; \ @@ -59,7 +60,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_2(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ xor64(p, tweak1_2); \ } while(0) @@ -67,7 +68,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_CHECK() \ do if (length < 43) \ { \ - fprintf(stderr, "Cryptonight variants need at least 43 bytes of data"); \ + fprintf(stderr, "Cryptonight variant 1 needs at least 43 bytes of data"); \ _exit(1); \ } while(0) @@ -75,7 +76,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_PORTABLE_INIT() \ uint8_t tweak1_2[8]; \ - do if (variant > 0) \ + do if (variant == 1) \ { \ VARIANT1_CHECK(); \ memcpy(&tweak1_2, &state.hs.b[192], sizeof(tweak1_2)); \ @@ -83,11 +84,119 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_INIT64() \ - if (variant > 0) \ + if (variant == 1) \ { \ VARIANT1_CHECK(); \ } \ - const uint64_t tweak1_2 = variant > 0 ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + const uint64_t tweak1_2 = (variant == 1) ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + +#define VARIANT2_INIT64() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \ + U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_PORTABLE_INIT() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + memcpy(b + AES_BLOCK_SIZE, state.hs.b + 64, AES_BLOCK_SIZE); \ + xor64(b + AES_BLOCK_SIZE, state.hs.b + 80); \ + xor64(b + AES_BLOCK_SIZE + 8, state.hs.b + 88); \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_SSE2(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const __m128i chunk1 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10))); \ + const __m128i chunk2 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20))); \ + const __m128i chunk3 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30))); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10)), _mm_add_epi64(chunk3, _b1)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20)), _mm_add_epi64(chunk1, _b)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30)), _mm_add_epi64(chunk2, _a)); \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_NEON(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const uint64x2_t chunk1 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x10))); \ + const uint64x2_t chunk2 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x20))); \ + const uint64x2_t chunk3 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x30))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x10)), vaddq_u64(chunk3, vreinterpretq_u64_u8(_b1))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x20)), vaddq_u64(chunk1, vreinterpretq_u64_u8(_b))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x30)), vaddq_u64(chunk2, vreinterpretq_u64_u8(_a))); \ + } while (0) + +#define VARIANT2_PORTABLE_SHUFFLE_ADD(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + uint64_t* chunk1 = U64((base_ptr) + ((offset) ^ 0x10)); \ + uint64_t* chunk2 = U64((base_ptr) + ((offset) ^ 0x20)); \ + uint64_t* chunk3 = U64((base_ptr) + ((offset) ^ 0x30)); \ + \ + const uint64_t chunk1_old[2] = { chunk1[0], chunk1[1] }; \ + \ + uint64_t b1[2]; \ + memcpy(b1, b + 16, 16); \ + chunk1[0] = chunk3[0] + b1[0]; \ + chunk1[1] = chunk3[1] + b1[1]; \ + \ + uint64_t a0[2]; \ + memcpy(a0, a, 16); \ + chunk3[0] = chunk2[0] + a0[0]; \ + chunk3[1] = chunk2[1] + a0[1]; \ + \ + uint64_t b0[2]; \ + memcpy(b0, b, 16); \ + chunk2[0] = chunk1_old[0] + b0[0]; \ + chunk2[1] = chunk1_old[1] + b0[1]; \ + } while (0) + +#define VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr) \ + ((uint64_t*)(b))[0] ^= division_result ^ (sqrt_result << 32); \ + { \ + const uint64_t dividend = ((uint64_t*)(ptr))[1]; \ + const uint32_t divisor = (((uint64_t*)(ptr))[0] + (uint32_t)(sqrt_result << 1)) | 0x80000001UL; \ + division_result = ((uint32_t)(dividend / divisor)) + \ + (((uint64_t)(dividend % divisor)) << 32); \ + } \ + const uint64_t sqrt_input = ((uint64_t*)(ptr))[0] + division_result + +#define VARIANT2_INTEGER_MATH_SSE2(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while(0) + +#if defined DBL_MANT_DIG && (DBL_MANT_DIG >= 50) + // double precision floating point type has enough bits of precision on current platform + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_FP64(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while (0) +#else + // double precision floating point type is not good enough on current platform + // fall back to the reference code (integer only) + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_REF(); \ + } while (0) +#endif #if !defined NO_AES && (defined(__x86_64__) || (defined(_MSC_VER) && defined(_WIN64))) // Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI @@ -164,19 +273,22 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp * This code is based upon an optimized implementation by dga. */ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _b = _mm_xor_si128(_b, _c); \ - _mm_store_si128(R128(&hp_state[j]), _b); \ + _mm_store_si128(R128(&hp_state[j]), _mm_xor_si128(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_INTEGER_MATH_SSE2(b, c); \ __mul(); \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ #if defined(_MSC_VER) @@ -492,7 +604,7 @@ void slow_hash_allocate_state(void) MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); #else #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ - defined(__DragonFly__) + defined(__DragonFly__) || defined(__NetBSD__) hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, 0, 0); #else @@ -570,10 +682,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - __m128i _a, _b, _c; + __m128i _a, _b, _b1, _c; uint64_t hi, lo; size_t i, j; @@ -599,6 +711,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -637,6 +750,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = _mm_load_si128(R128(b)); + _b1 = _mm_load_si128(R128(b) + 1); // Two independent versions, one with AES, one without, to ensure that // the useAes test is only performed once, not every iteration. if(useAes) @@ -761,19 +875,22 @@ union cn_slow_hash_state _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - _b = veorq_u8(_b, _c); \ - vst1q_u8(&hp_state[j], _b); \ + vst1q_u8(&hp_state[j], veorq_u8(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ __mul(); \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ @@ -912,10 +1029,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - uint8x16_t _a, _b, _c, zero = {0}; + uint8x16_t _a, _b, _b1, _c, zero = {0}; uint64_t hi, lo; size_t i, j; @@ -936,6 +1053,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -959,7 +1077,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = vld1q_u8((const uint8_t *)b); - + _b1 = vld1q_u8(((const uint8_t *)b) + AES_BLOCK_SIZE); for(i = 0; i < ITER / 2; i++) { @@ -1075,6 +1193,11 @@ __asm__ __volatile__( #endif /* !aarch64 */ #endif // NO_OPTIMIZED_MULTIPLY_ON_ARM +STATIC INLINE void copy_block(uint8_t* dst, const uint8_t* src) +{ + memcpy(dst, src, AES_BLOCK_SIZE); +} + STATIC INLINE void sum_half_blocks(uint8_t* a, const uint8_t* b) { uint64_t a0, a1, b0, b1; @@ -1109,7 +1232,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c[AES_BLOCK_SIZE]; + uint8_t c1[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; uint8_t aes_key[AES_KEY_SIZE]; RDATA_ALIGN16 uint8_t expandedKey[256]; @@ -1138,11 +1263,12 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } memcpy(text, state.init, INIT_SIZE_BYTE); - VARIANT1_INIT64(); - aes_ctx = (oaes_ctx *) oaes_alloc(); oaes_key_import_data(aes_ctx, state.hs.b, AES_KEY_SIZE); + VARIANT1_INIT64(); + VARIANT2_INIT64(); + // use aligned data memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len); for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) @@ -1163,23 +1289,33 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #define state_index(x) ((*(uint32_t *) x) & MASK) // Iteration 1 - p = &long_state[state_index(a)]; + j = state_index(a); + p = &long_state[j]; aesb_single_round(p, p, a); + copy_block(c1, p); - xor_blocks(b, p); - swap_blocks(b, p); - swap_blocks(a, b); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + xor_blocks(p, b); VARIANT1_1(p); // Iteration 2 - p = &long_state[state_index(a)]; - - mul(a, p, d); - sum_half_blocks(b, d); - swap_blocks(b, p); - xor_blocks(b, p); - swap_blocks(a, b); - VARIANT1_2(U64(p) + 1); + j = state_index(c1); + p = &long_state[j]; + copy_block(c, p); + + VARIANT2_PORTABLE_INTEGER_MATH(c, c1); + mul(c1, c, d); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + sum_half_blocks(a, d); + swap_blocks(a, c); + xor_blocks(a, c); + VARIANT1_2(U64(c) + 1); + copy_block(p, c); + + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1298,8 +1434,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int union cn_slow_hash_state state; uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; - uint8_t c[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c1[AES_BLOCK_SIZE]; + uint8_t c2[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; size_t i, j; uint8_t aes_key[AES_KEY_SIZE]; @@ -1315,6 +1452,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int aes_ctx = (oaes_ctx *) oaes_alloc(); VARIANT1_PORTABLE_INIT(); + VARIANT2_PORTABLE_INIT(); oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { @@ -1324,9 +1462,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } - for (i = 0; i < 16; i++) { - a[i] = state.k[ i] ^ state.k[32 + i]; - b[i] = state.k[16 + i] ^ state.k[48 + i]; + for (i = 0; i < AES_BLOCK_SIZE; i++) { + a[i] = state.k[ i] ^ state.k[AES_BLOCK_SIZE * 2 + i]; + b[i] = state.k[AES_BLOCK_SIZE + i] ^ state.k[AES_BLOCK_SIZE * 3 + i]; } for (i = 0; i < ITER / 2; i++) { @@ -1335,26 +1473,32 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int * next address <-+ */ /* Iteration 1 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - aesb_single_round(c, c, a); - xor_blocks(b, c); - swap_blocks(b, c); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); - VARIANT1_1(&long_state[j * AES_BLOCK_SIZE]); + j = e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c1, &long_state[j]); + aesb_single_round(c1, c1, a); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + copy_block(&long_state[j], c1); + xor_blocks(&long_state[j], b); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + VARIANT1_1(&long_state[j]); /* Iteration 2 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - mul(a, c, d); - sum_half_blocks(b, d); - swap_blocks(b, c); - xor_blocks(b, c); - VARIANT1_2(c + 8); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); + j = e2i(c1, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c2, &long_state[j]); + VARIANT2_PORTABLE_INTEGER_MATH(c2, c1); + mul(c1, c2, d); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + swap_blocks(a, c1); + sum_half_blocks(c1, d); + swap_blocks(c1, c2); + xor_blocks(c1, c2); + VARIANT1_2(c2 + 8); + copy_block(&long_state[j], c2); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, a); + copy_block(a, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 57c38b86b..b2dc3ffb2 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -36,7 +36,8 @@ #ifdef _MSC_VER #include <malloc.h> -#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) +#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) \ + && !defined(__NetBSD__) #include <alloca.h> #else #include <stdlib.h> diff --git a/src/crypto/variant2_int_sqrt.h b/src/crypto/variant2_int_sqrt.h new file mode 100644 index 000000000..b405bb798 --- /dev/null +++ b/src/crypto/variant2_int_sqrt.h @@ -0,0 +1,163 @@ +#ifndef VARIANT2_INT_SQRT_H +#define VARIANT2_INT_SQRT_H + +#include <math.h> +#include <float.h> + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2() \ + do { \ + const __m128i exp_double_bias = _mm_set_epi64x(0, 1023ULL << 52); \ + __m128d x = _mm_castsi128_pd(_mm_add_epi64(_mm_cvtsi64_si128(sqrt_input >> 12), exp_double_bias)); \ + x = _mm_sqrt_sd(_mm_setzero_pd(), x); \ + sqrt_result = (uint64_t)(_mm_cvtsi128_si64(_mm_sub_epi64(_mm_castpd_si128(x), exp_double_bias))) >> 19; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_FP64() \ + do { \ + sqrt_result = sqrt(sqrt_input + 18446744073709551616.0) * 2.0 - 8589934592.0; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_REF() \ + sqrt_result = integer_square_root_v2(sqrt_input) + +// Reference implementation of the integer square root for Cryptonight variant 2 +// Computes integer part of "sqrt(2^64 + n) * 2 - 2^33" +// +// In other words, given 64-bit unsigned integer n: +// 1) Write it as x = 1.NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN000... in binary (1 <= x < 2, all 64 bits of n are used) +// 2) Calculate sqrt(x) = 1.0RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR... (1 <= sqrt(x) < sqrt(2), so it will always start with "1.0" in binary) +// 3) Take 32 bits that come after "1.0" and return them as a 32-bit unsigned integer, discard all remaining bits +// +// Some sample inputs and outputs: +// +// Input | Output | Exact value of "sqrt(2^64 + n) * 2 - 2^33" +// -----------------|------------|------------------------------------------- +// 0 | 0 | 0 +// 2^32 | 0 | 0.99999999994179233909330885695244... +// 2^32 + 1 | 1 | 1.0000000001746229827200734316305... +// 2^50 | 262140 | 262140.00012206565608606978175873... +// 2^55 + 20963331 | 8384515 | 8384515.9999999997673963974959744... +// 2^55 + 20963332 | 8384516 | 8384516 +// 2^62 + 26599786 | 1013904242 | 1013904242.9999999999479374853545... +// 2^62 + 26599787 | 1013904243 | 1013904243.0000000001561875439364... +// 2^64 - 1 | 3558067407 | 3558067407.9041987696409179931096... + +// The reference implementation as it is now uses only unsigned int64 arithmetic, so it can't have undefined behavior +// It was tested once for all edge cases and confirmed correct +static inline uint32_t integer_square_root_v2(uint64_t n) +{ + uint64_t r = 1ULL << 63; + + for (uint64_t bit = 1ULL << 60; bit; bit >>= 2) + { + const bool b = (n < r + bit); + const uint64_t n_next = n - (r + bit); + const uint64_t r_next = r + bit * 2; + n = b ? n : n_next; + r = b ? r : r_next; + r >>= 1; + } + + return r * 2 + ((n > r) ? 1 : 0); +} + +/* +VARIANT2_INTEGER_MATH_SQRT_FIXUP checks that "r" is an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33" and adds or subtracts 1 if needed +It's hard to understand how it works, so here is a full calculation of formulas used in VARIANT2_INTEGER_MATH_SQRT_FIXUP + +The following inequalities must hold for r if it's an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33": +1) r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +2) r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 + +We need to check them using only unsigned integer arithmetic to avoid rounding errors and undefined behavior + +First inequality: r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 2^33 <= sqrt(2^64 + sqrt_input) * 2 +r/2 + 2^32 <= sqrt(2^64 + sqrt_input) +(r/2 + 2^32)^2 <= 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b)/2 + 2^32)^2 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 + 2^64 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 <= sqrt_input +(s*2+b)^2/4 + 2^32*r <= sqrt_input +(s^2*4+2*s*2*b+b^2)/4 + 2^32*r <= sqrt_input +s^2+s*b+b^2/4 + 2^32*r <= sqrt_input +s*(s+b) + b^2/4 + 2^32*r <= sqrt_input + +Let r2 = s*(s+b) + r*2^32 +r2 + b^2/4 <= sqrt_input + +If this inequality doesn't hold, then we must decrement r: IF "r2 + b^2/4 > sqrt_input" THEN r = r - 1 + +b can be 0 or 1 +If b is 0 then we need to compare "r2 > sqrt_input" +If b is 1 then b^2/4 = 0.25, so we need to compare "r2 + 0.25 > sqrt_input" +Since both r2 and sqrt_input are integers, we can safely replace it with "r2 + 1 > sqrt_input" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + b > sqrt_input" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + b", so it's safe to compare with sqrt_input: +r2 + b = s*(s+b) + r*2^32 + b +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067407*2^32 + 1 = 18446744068217447385 < 2^64 + +Second inequality: r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 1 + 2^33 > sqrt(2^64 + sqrt_input) * 2 +((r+1)/2 + 2^32)^2 > 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b+1)/2 + 2^32)^2 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + 2*(s*2+b+1)/2*2^32 + 2^64 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + (s*2+b+1)*2^32 > sqrt_input +(s*2+b+1)^2/4 + (r+1)*2^32 > sqrt_input +(s*2+(b+1))^2/4 + r*2^32 + 2^32 > sqrt_input +(s^2*4+2*s*2*(b+1)+(b+1)^2)/4 + r*2^32 + 2^32 > sqrt_input +s^2+s*(b+1)+(b+1)^2/4 + r*2^32 + 2^32 > sqrt_input +s*(s+b) + s + (b+1)^2/4 + r*2^32 + 2^32 > sqrt_input + +Let r2 = s*(s+b) + r*2^32 + +r2 + s + (b+1)^2/4 + 2^32 > sqrt_input +r2 + 2^32 + (b+1)^2/4 > sqrt_input - s + +If this inequality doesn't hold, then we must decrement r: IF "r2 + 2^32 + (b+1)^2/4 <= sqrt_input - s" THEN r = r - 1 +b can be 0 or 1 +If b is 0 then we need to compare "r2 + 2^32 + 1/4 <= sqrt_input - s" which is equal to "r2 + 2^32 < sqrt_input - s" because all numbers here are integers +If b is 1 then (b+1)^2/4 = 1, so we need to compare "r2 + 2^32 + 1 <= sqrt_input - s" which is also equal to "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + 2^32": +r2 + 2^32 = s*(s+b) + r*2^32 + 2^32 = s*(s+b) + (r+1)*2^32 +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067408*2^32 = 18446744072512414680 < 2^64 + +There will be no integer overflow when calculating "sqrt_input - s", i.e. "sqrt_input >= s" at all times: +s = trunc(r/2) = trunc(sqrt(2^64 + sqrt_input) - 2^32) < sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input > sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input + 2^32 - 1 > sqrt(2^64 + sqrt_input) +(sqrt_input + 2^32 - 1)^2 > sqrt_input + 2^64 +sqrt_input^2 + 2*sqrt_input*(2^32 - 1) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 2) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + (2^32-1)^2 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + 2^64-2^33+1 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) - 2^33 + 1 > 0 +This inequality is true if sqrt_input > 1 and it's easy to check that s = 0 if sqrt_input is 0 or 1, so there will be no integer overflow +*/ + +#define VARIANT2_INTEGER_MATH_SQRT_FIXUP(r) \ + do { \ + const uint64_t s = r >> 1; \ + const uint64_t b = r & 1; \ + const uint64_t r2 = (uint64_t)(s) * (s + b) + (r << 32); \ + r += ((r2 + b > sqrt_input) ? -1 : 0) + ((r2 + (1ULL << 32) < sqrt_input - s) ? 1 : 0); \ + } while(0) + +#endif diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index d50a9df67..21445959d 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -27,9 +27,13 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. if(APPLE) - find_library(IOKIT_LIBRARY IOKit) - mark_as_advanced(IOKIT_LIBRARY) - list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + if(DEPENDS) + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework ApplicationServices -framework AppKit -framework IOKit") + else() + find_library(IOKIT_LIBRARY IOKit) + mark_as_advanced(IOKIT_LIBRARY) + list(APPEND EXTRA_LIBRARIES ${IOKIT_LIBRARY}) + endif() endif() set(cryptonote_basic_sources diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index aac6ec22b..e891a748d 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -44,6 +44,9 @@ extern "C" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "account" +#define KEYS_ENCRYPTION_SALT 'k' + + using namespace std; DISABLE_VS_WARNINGS(4244 4345) @@ -60,7 +63,70 @@ DISABLE_VS_WARNINGS(4244 4345) m_device = &hwdev; MCDEBUG("device", "account_keys::set_device device type: "<<typeid(hwdev).name()); } + //----------------------------------------------------------------- + static void derive_key(const crypto::chacha_key &base_key, crypto::chacha_key &key) + { + static_assert(sizeof(base_key) == sizeof(crypto::hash), "chacha key and hash should be the same size"); + epee::mlocked<tools::scrubbed_arr<char, sizeof(base_key)+1>> data; + memcpy(data.data(), &base_key, sizeof(base_key)); + data[sizeof(base_key)] = KEYS_ENCRYPTION_SALT; + crypto::generate_chacha_key(data.data(), sizeof(data), key, 1); + } + //----------------------------------------------------------------- + static epee::wipeable_string get_key_stream(const crypto::chacha_key &base_key, const crypto::chacha_iv &iv, size_t bytes) + { + // derive a new key + crypto::chacha_key key; + derive_key(base_key, key); + // chacha + epee::wipeable_string buffer0(std::string(bytes, '\0')); + epee::wipeable_string buffer1 = buffer0; + crypto::chacha20(buffer0.data(), buffer0.size(), key, iv, buffer1.data()); + return buffer1; + } + //----------------------------------------------------------------- + void account_keys::xor_with_key_stream(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * (2 + m_multisig_keys.size())); + const char *ptr = key_stream.data(); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_spend_secret_key.data[i] ^= *ptr++; + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + for (crypto::secret_key &k: m_multisig_keys) + { + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + k.data[i] ^= *ptr++; + } + } + //----------------------------------------------------------------- + void account_keys::encrypt(const crypto::chacha_key &key) + { + m_encryption_iv = crypto::rand<crypto::chacha_iv>(); + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::decrypt(const crypto::chacha_key &key) + { + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::encrypt_viewkey(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * 2); + const char *ptr = key_stream.data(); + ptr += sizeof(crypto::secret_key); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + } + //----------------------------------------------------------------- + void account_keys::decrypt_viewkey(const crypto::chacha_key &key) + { + encrypt_viewkey(key); + } //----------------------------------------------------------------- account_base::account_base() { @@ -131,10 +197,14 @@ DISABLE_VS_WARNINGS(4244 4345) //----------------------------------------------------------------- void account_base::create_from_device(const std::string &device_name) { - hw::device &hwdev = hw::get_device(device_name); - m_keys.set_device(hwdev); hwdev.set_name(device_name); + create_from_device(hwdev); + } + + void account_base::create_from_device(hw::device &hwdev) + { + m_keys.set_device(hwdev); MCDEBUG("ledger", "device type: "<<typeid(hwdev).name()); hwdev.init(); hwdev.connect(); @@ -157,7 +227,7 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey) { crypto::secret_key fake; - memset(&unwrap(fake), 0, sizeof(fake)); + memset(&unwrap(unwrap(fake)), 0, sizeof(fake)); create_from_keys(address, fake, viewkey); } //----------------------------------------------------------------- diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index b5d119c46..98bba55b1 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -44,18 +44,29 @@ namespace cryptonote crypto::secret_key m_view_secret_key; std::vector<crypto::secret_key> m_multisig_keys; hw::device *m_device = &hw::get_device("default"); + crypto::chacha_iv m_encryption_iv; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(m_account_address) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_secret_key) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_secret_key) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_multisig_keys) + const crypto::chacha_iv default_iv{{0, 0, 0, 0, 0, 0, 0, 0}}; + KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(m_encryption_iv, default_iv) END_KV_SERIALIZE_MAP() account_keys& operator=(account_keys const&) = default; + void encrypt(const crypto::chacha_key &key); + void decrypt(const crypto::chacha_key &key); + void encrypt_viewkey(const crypto::chacha_key &key); + void decrypt_viewkey(const crypto::chacha_key &key); + hw::device& get_device() const ; void set_device( hw::device &hwdev) ; + + private: + void xor_with_key_stream(const crypto::chacha_key &key); }; /************************************************************************/ @@ -66,7 +77,8 @@ namespace cryptonote public: account_base(); crypto::secret_key generate(const crypto::secret_key& recovery_key = crypto::secret_key(), bool recover = false, bool two_random = false); - void create_from_device(const std::string &device_name) ; + void create_from_device(const std::string &device_name); + void create_from_device(hw::device &hwdev); void create_from_keys(const cryptonote::account_public_address& address, const crypto::secret_key& spendkey, const crypto::secret_key& viewkey); void create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey); bool make_multisig(const crypto::secret_key &view_secret_key, const crypto::secret_key &spend_secret_key, const crypto::public_key &spend_public_key, const std::vector<crypto::secret_key> &multisig_keys); @@ -87,6 +99,11 @@ namespace cryptonote void forget_spend_key(); const std::vector<crypto::secret_key> &get_multisig_keys() const { return m_keys.m_multisig_keys; } + void encrypt_keys(const crypto::chacha_key &key) { m_keys.encrypt(key); } + void decrypt_keys(const crypto::chacha_key &key) { m_keys.decrypt(key); } + void encrypt_viewkey(const crypto::chacha_key &key) { m_keys.encrypt_viewkey(key); } + void decrypt_viewkey(const crypto::chacha_key &key) { m_keys.decrypt_viewkey(key); } + template <class t_archive> inline void serialize(t_archive &a, const unsigned int /*ver*/) { diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index cff23695f..b18ef1c5c 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -67,7 +67,7 @@ namespace cryptonote { /* Cryptonote helper functions */ /************************************************************************/ //----------------------------------------------------------------------------------------------- - size_t get_min_block_size(uint8_t version) + size_t get_min_block_weight(uint8_t version) { if (version < 2) return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; @@ -86,7 +86,7 @@ namespace cryptonote { return CRYPTONOTE_MAX_TX_SIZE; } //----------------------------------------------------------------------------------------------- - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { static_assert(DIFFICULTY_TARGET_V2%60==0&&DIFFICULTY_TARGET_V1%60==0,"difficulty targets must be a multiple of 60"); const int target = version < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; const int target_minutes = target / 60; @@ -98,37 +98,37 @@ namespace cryptonote { base_reward = FINAL_SUBSIDY_PER_MINUTE*target_minutes; } - uint64_t full_reward_zone = get_min_block_size(version); + uint64_t full_reward_zone = get_min_block_weight(version); //make it soft - if (median_size < full_reward_zone) { - median_size = full_reward_zone; + if (median_weight < full_reward_zone) { + median_weight = full_reward_zone; } - if (current_block_size <= median_size) { + if (current_block_weight <= median_weight) { reward = base_reward; return true; } - if(current_block_size > 2 * median_size) { - MERROR("Block cumulative size is too big: " << current_block_size << ", expected less than " << 2 * median_size); + if(current_block_weight > 2 * median_weight) { + MERROR("Block cumulative weight is too big: " << current_block_weight << ", expected less than " << 2 * median_weight); return false; } - assert(median_size < std::numeric_limits<uint32_t>::max()); - assert(current_block_size < std::numeric_limits<uint32_t>::max()); + assert(median_weight < std::numeric_limits<uint32_t>::max()); + assert(current_block_weight < std::numeric_limits<uint32_t>::max()); uint64_t product_hi; // BUGFIX: 32-bit saturation bug (e.g. ARM7), the result was being // treated as 32-bit by default. - uint64_t multiplicand = 2 * median_size - current_block_size; - multiplicand *= current_block_size; + uint64_t multiplicand = 2 * median_weight - current_block_weight; + multiplicand *= current_block_weight; uint64_t product_lo = mul128(base_reward, multiplicand, &product_hi); uint64_t reward_hi; uint64_t reward_lo; - div128_32(product_hi, product_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); - div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); + div128_32(product_hi, product_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); + div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); assert(0 == reward_hi); assert(reward_lo < base_reward); diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index f59785021..c804a88fa 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -86,10 +86,10 @@ namespace cryptonote { /************************************************************************/ /* Cryptonote helper functions */ /************************************************************************/ - size_t get_min_block_size(uint8_t version); + size_t get_min_block_weight(uint8_t version); size_t get_max_block_size(); size_t get_max_tx_size(); - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); uint8_t get_account_address_checksum(const public_address_outer_blob& bl); uint8_t get_account_integrated_address_checksum(const public_integrated_address_outer_blob& bl); diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 143133163..0725a2bb8 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -295,7 +295,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +323,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,7 +337,7 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeSimpleBulletproof) + if (x.type == rct::RCTTypeBulletproof) a & x.p.pseudoOuts; } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 428be1c9c..5fcfa33f6 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -135,23 +135,41 @@ namespace cryptonote return false; } for (size_t n = 0; n < tx.rct_signatures.outPk.size(); ++n) + { + if (tx.vout[n].target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Unsupported output type in tx " << get_transaction_hash(tx)); + return false; + } rv.outPk[n].dest = rct::pk2rct(boost::get<txout_to_key>(tx.vout[n].target).key); + } if (!base_only) { - const bool bulletproof = rv.type == rct::RCTTypeFullBulletproof || rv.type == rct::RCTTypeSimpleBulletproof; + const bool bulletproof = rct::is_rct_bulletproof(rv.type); if (bulletproof) { - if (rv.p.bulletproofs.size() != tx.vout.size()) + if (rv.p.bulletproofs.size() != 1) { LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs size in tx " << get_transaction_hash(tx)); return false; } - for (size_t n = 0; n < rv.outPk.size(); ++n) + if (rv.p.bulletproofs[0].L.size() < 6) + { + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs L size in tx " << get_transaction_hash(tx)); + return false; + } + const size_t max_outputs = 1 << (rv.p.bulletproofs[0].L.size() - 6); + if (max_outputs < tx.vout.size()) { - rv.p.bulletproofs[n].V.resize(1); - rv.p.bulletproofs[n].V[0] = rv.outPk[n].mask; + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs max outputs in tx " << get_transaction_hash(tx)); + return false; } + const size_t n_amounts = tx.vout.size(); + CHECK_AND_ASSERT_MES(n_amounts == rv.outPk.size(), false, "Internal error filling out V"); + rv.p.bulletproofs[0].V.resize(n_amounts); + for (size_t i = 0; i < n_amounts; ++i) + rv.p.bulletproofs[0].V[i] = rct::scalarmultKey(rv.outPk[i].mask, rct::INV_EIGHT); } } } @@ -201,15 +219,25 @@ namespace cryptonote { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); bool r = hwdev.generate_key_derivation(tx_public_key, ack.m_view_secret_key, recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + memcpy(&recv_derivation, rct::identity().bytes, sizeof(recv_derivation)); + } std::vector<crypto::key_derivation> additional_recv_derivations; for (size_t i = 0; i < additional_tx_public_keys.size(); ++i) { crypto::key_derivation additional_recv_derivation = AUTO_VAL_INIT(additional_recv_derivation); r = hwdev.generate_key_derivation(additional_tx_public_keys[i], ack.m_view_secret_key, additional_recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); - additional_recv_derivations.push_back(additional_recv_derivation); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); + } + else + { + additional_recv_derivations.push_back(additional_recv_derivation); + } } boost::optional<subaddress_receive_info> subaddr_recv_info = is_out_to_acc_precomp(subaddresses, out_key, recv_derivation, additional_recv_derivations, real_output_index,hwdev); @@ -318,6 +346,37 @@ namespace cryptonote return string_tools::get_xtype_from_string(amount, str_amount); } //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size) + { + if (tx.version < 2) + return blob_size; + const rct::rctSig &rv = tx.rct_signatures; + if (!rct::is_rct_bulletproof(rv.type)) + return blob_size; + const size_t n_outputs = tx.vout.size(); + if (n_outputs <= 2) + return blob_size; + const uint64_t bp_base = 368; + const size_t n_padded_outputs = rct::n_bulletproof_max_amounts(rv.p.bulletproofs); + size_t nlr = 0; + for (const auto &bp: rv.p.bulletproofs) + nlr += bp.L.size() * 2; + const size_t bp_size = 32 * (9 + nlr); + CHECK_AND_ASSERT_THROW_MES_L1(bp_base * n_padded_outputs >= bp_size, "Invalid bulletproof clawback"); + const uint64_t bp_clawback = (bp_base * n_padded_outputs - bp_size) * 4 / 5; + CHECK_AND_ASSERT_THROW_MES_L1(bp_clawback <= std::numeric_limits<uint64_t>::max() - blob_size, "Weight overflow"); + return blob_size + bp_clawback; + } + //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx) + { + std::ostringstream s; + binary_archive<true> a(s); + ::serialization::serialize(a, const_cast<transaction&>(tx)); + const cryptonote::blobdata blob = s.str(); + return get_transaction_weight(tx, blob.size()); + } + //--------------------------------------------------------------- bool get_tx_fee(const transaction& tx, uint64_t & fee) { if (tx.version > 1) diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8a5296d5b..bf71eb591 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -117,6 +117,8 @@ namespace cryptonote bool check_inputs_types_supported(const transaction& tx); bool check_outs_valid(const transaction& tx); bool parse_amount(uint64_t& amount, const std::string& str_amount); + uint64_t get_transaction_weight(const transaction &tx); + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size); bool check_money_overflow(const transaction& tx); bool check_outs_overflow(const transaction& tx); diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index a0dcf2df1..a6858ce7c 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -65,9 +65,11 @@ #define FEE_PER_KB_OLD ((uint64_t)10000000000) // pow(10, 10) #define FEE_PER_KB ((uint64_t)2000000000) // 2 * pow(10, 9) +#define FEE_PER_BYTE ((uint64_t)300000) #define DYNAMIC_FEE_PER_KB_BASE_FEE ((uint64_t)2000000000) // 2 * pow(10,9) #define DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD ((uint64_t)10000000000000) // 10 * pow(10,12) #define DYNAMIC_FEE_PER_KB_BASE_FEE_V5 ((uint64_t)2000000000 * (uint64_t)CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 / CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5) +#define DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT ((uint64_t)3000) #define ORPHANED_BLOCKS_MAX_COUNT 100 @@ -133,13 +135,17 @@ #define HF_VERSION_DYNAMIC_FEE 4 #define HF_VERSION_MIN_MIXIN_4 6 #define HF_VERSION_MIN_MIXIN_6 7 +#define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 +#define HF_VERSION_PER_BYTE_FEE 8 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 #define HASH_OF_HASHES_STEP 256 -#define DEFAULT_TXPOOL_MAX_SIZE 648000000ull // 3 days at 300000, in bytes +#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes + +#define BULLETPROOF_MAX_OUTPUTS 16 // New constants are intended to go here namespace config diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 6e35744e6..3751d6473 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -109,6 +109,12 @@ static const struct { // version 7 starts from block 1546000, which is on or around the 6th of April, 2018. Fork time finalised on 2018-03-17. { 7, 1546000, 0, 1521303150 }, + + // version 8 starts from block 1685555, which is on or around the 18th of October, 2018. Fork time finalised on 2018-09-02. + { 8, 1685555, 0, 1535889547 }, + + // version 9 starts from block 1686275, which is on or around the 19th of October, 2018. Fork time finalised on 2018-09-02. + { 9, 1686275, 0, 1535889548 }, }; static const uint64_t mainnet_hard_fork_version_1_till = 1009826; @@ -132,6 +138,7 @@ static const struct { { 6, 971400, 0, 1501709789 }, { 7, 1057027, 0, 1512211236 }, { 8, 1057058, 0, 1515967497 }, + { 9, 1057778, 0, 1515967498 }, }; static const uint64_t testnet_hard_fork_version_1_till = 624633; @@ -155,10 +162,11 @@ static const struct { //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : - m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), m_current_block_cumul_sz_median(0), - m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false), + m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), + m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), m_difficulty_for_next_block_top_hash(crypto::null_hash), - m_difficulty_for_next_block(1) + m_difficulty_for_next_block(1), + m_btc_valid(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -481,7 +489,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); } - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return true; } //------------------------------------------------------------------ @@ -630,8 +638,9 @@ block Blockchain::pop_block_from_blockchain() m_blocks_txs_check.clear(); m_check_txin_table.clear(); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); + invalidate_block_template_cache(); return popped_block; } @@ -642,12 +651,13 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) CRITICAL_REGION_LOCAL(m_blockchain_lock); m_timestamps_and_difficulties_height = 0; m_alternative_chains.clear(); + invalidate_block_template_cache(); m_db->reset(); m_hardfork->init(); block_verification_context bvc = boost::value_initialized<block_verification_context>(); add_new_block(b, bvc); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ @@ -1110,7 +1120,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) } //------------------------------------------------------------------ // This function validates the miner transaction reward -bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) { LOG_PRINT_L3("Blockchain::" << __func__); //validate reward @@ -1128,11 +1138,11 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } } - std::vector<size_t> last_blocks_sizes; - get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward, version)) + std::vector<size_t> last_blocks_weights; + get_last_n_blocks_weights(last_blocks_weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_weights), cumulative_block_weight, already_generated_coins, base_reward, version)) { - MERROR_VER("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + MERROR_VER("block weight " << cumulative_block_weight << " is bigger than allowed for this blockchain"); return false; } if(base_reward + fee < money_in_use) @@ -1162,8 +1172,8 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return true; } //------------------------------------------------------------------ -// get the block sizes of the last <count> blocks, and return by reference <sz>. -void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const +// get the block weights of the last <count> blocks, and return by reference <sz>. +void Blockchain::get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1174,26 +1184,26 @@ void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) return; m_db->block_txn_start(true); - // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) + // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); - sz.reserve(sz.size() + h - start_offset); + weights.reserve(weights.size() + h - start_offset); for(size_t i = start_offset; i < h; i++) { - sz.push_back(m_db->get_block_size(i)); + weights.push_back(m_db->get_block_weight(i)); } m_db->block_txn_stop(); } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_limit() const +uint64_t Blockchain::get_current_cumulative_block_weight_limit() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_limit; + return m_current_block_cumul_weight_limit; } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_median() const +uint64_t Blockchain::get_current_cumulative_block_weight_median() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_median; + return m_current_block_cumul_weight_median; } //------------------------------------------------------------------ //TODO: This function only needed minor modification to work with BlockchainDB, @@ -1210,11 +1220,28 @@ uint64_t Blockchain::get_current_cumulative_blocksize_median() const bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) { LOG_PRINT_L3("Blockchain::" << __func__); - size_t median_size; + size_t median_weight; uint64_t already_generated_coins; + uint64_t pool_cookie; CRITICAL_REGION_BEGIN(m_blockchain_lock); height = m_db->height(); + if (m_btc_valid) { + // The pool cookie is atomic. The lack of locking is OK, as if it changes + // just as we compare it, we'll just use a slightly old template, but + // this would be the case anyway if we'd lock, and the change happened + // just after the block template was created + if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce && m_btc_pool_cookie == m_tx_pool.cookie()) { + MDEBUG("Using cached template"); + m_btc.timestamp = time(NULL); // update timestamp unconditionally + b = m_btc; + diffic = m_btc_difficulty; + expected_reward = m_btc_expected_reward; + return true; + } + MDEBUG("Not using cached template: address " << (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address))) << ", nonce " << (m_btc_nonce == ex_nonce) << ", cookie " << (m_btc_pool_cookie == m_tx_pool.cookie())); + invalidate_block_template_cache(); + } b.major_version = m_hardfork->get_current_version(); b.minor_version = m_hardfork->get_ideal_version(); @@ -1230,19 +1257,20 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m diffic = get_difficulty_for_next_block(); CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead."); - median_size = m_current_block_cumul_sz_limit / 2; + median_weight = m_current_block_cumul_weight_limit / 2; already_generated_coins = m_db->get_block_already_generated_coins(height - 1); CRITICAL_REGION_END(); - size_t txs_size; + size_t txs_weight; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee, expected_reward, m_hardfork->get_current_version())) + if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, m_hardfork->get_current_version())) { return false; } + pool_cookie = m_tx_pool.cookie(); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - size_t real_txs_size = 0; + size_t real_txs_weight = 0; uint64_t real_fee = 0; CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); for(crypto::hash &cur_hash: b.tx_hashes) @@ -1254,11 +1282,11 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m continue; } tx_memory_pool::tx_details &cur_tx = cur_res->second; - real_txs_size += cur_tx.blob_size; + real_txs_weight += cur_tx.weight; real_fee += cur_tx.fee; - if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) + if (cur_tx.weight != get_transaction_weight(cur_tx.tx)) { - LOG_ERROR("Creating block template: error: invalid transaction size"); + LOG_ERROR("Creating block template: error: invalid transaction weight"); } if (cur_tx.tx.version == 1) { @@ -1280,9 +1308,9 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } } } - if (txs_size != real_txs_size) + if (txs_weight != real_txs_weight) { - LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + LOG_ERROR("Creating block template: error: wrongly calculated transaction weight"); } if (fee != real_fee) { @@ -1290,71 +1318,73 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } CRITICAL_REGION_END(); MDEBUG("Creating block template: height " << height << - ", median size " << median_size << + ", median weight " << median_weight << ", already generated coins " << already_generated_coins << - ", transaction size " << txs_size << + ", transaction weight " << txs_weight << ", fee " << fee); #endif /* - two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know - block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + two-phase miner transaction generation: we don't know exact block weight until we prepare block, but we don't know reward until we know + block weight, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block weight */ - //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob weight uint8_t hf_version = m_hardfork->get_current_version(); size_t max_outs = hf_version >= 4 ? 1 : 11; - bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + bool r = construct_miner_tx(height, median_weight, already_generated_coins, txs_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, first chance"); - size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); + size_t cumulative_weight = txs_weight + get_transaction_weight(b.miner_tx); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << - ", cumulative size " << cumulative_size); + MDEBUG("Creating block template: miner tx weight " << get_transaction_weight(b.miner_tx) << + ", cumulative weight " << cumulative_weight); #endif for (size_t try_count = 0; try_count != 10; ++try_count) { - r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + r = construct_miner_tx(height, median_weight, already_generated_coins, cumulative_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, second chance"); - size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); - if (coinbase_blob_size > cumulative_size - txs_size) + size_t coinbase_weight = get_transaction_weight(b.miner_tx); + if (coinbase_weight > cumulative_weight - txs_weight) { - cumulative_size = txs_size + coinbase_blob_size; + cumulative_weight = txs_weight + coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is greater than before"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is greater than before"); #endif continue; } - if (coinbase_blob_size < cumulative_size - txs_size) + if (coinbase_weight < cumulative_weight - txs_weight) { - size_t delta = cumulative_size - txs_size - coinbase_blob_size; + size_t delta = cumulative_weight - txs_weight - coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << txs_size + coinbase_blob_size << + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << txs_weight + coinbase_weight << " is less than before, adding " << delta << " zero bytes"); #endif b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight + 1 == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " + 1 is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_weight MDEBUG("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1); - cumulative_size += delta - 1; + cumulative_weight += delta - 1; continue; } MDEBUG("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count); } } - CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is now good"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is now good"); #endif + + cache_block_template(b, miner_address, ex_nonce, diffic, expected_reward, pool_cookie); return true; } LOG_ERROR("Failed to create_block_template with " << 10 << " tries"); @@ -1693,17 +1723,6 @@ size_t Blockchain::get_alternative_blocks_count() const //------------------------------------------------------------------ // This function adds the output specified by <amount, i> to the result_outs container // unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; -} - uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const { uint64_t num_outs = m_db->get_num_outputs(amount); @@ -1721,74 +1740,6 @@ uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const return num_outs; } -std::vector<uint64_t> Blockchain::get_random_outputs(uint64_t amount, uint64_t count) const -{ - uint64_t num_outs = get_num_mature_outputs(amount); - - std::vector<uint64_t> indices; - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if tx is unlocked, add output to indices - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - else - { - // while we still need more mixins - while (indices.size() < count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - - return indices; -} - crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_index) const { output_data_t data = m_db->get_output_key(amount, global_index); @@ -1796,169 +1747,6 @@ crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_i } //------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - for (uint64_t amount : req.amounts) - { - // create outs_for_amount struct and populate amount field - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); - result_outs.amount = amount; - - std::vector<uint64_t> indices = get_random_outputs(amount, req.outs_count); - - for (auto i : indices) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oe = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - - oe.global_amount_index = i; - oe.out_key = get_output_key(amount, i); - } - } - return true; -} -//------------------------------------------------------------------ -// This function adds the ringct output at index i to the list -// unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry& oen = *outs.insert(outs.end(), COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry()); - oen.amount = amount; - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; - oen.commitment = data.commitment; -} -//------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - auto num_outs = m_db->get_num_outputs(0); - // ensure we don't include outputs that aren't yet eligible to be used - // outpouts are sorted by height - while (num_outs > 0) - { - const tx_out_index toi = m_db->get_output_tx_and_index(0, num_outs - 1); - const uint64_t height = m_db->get_tx_block_height(toi.first); - if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height()) - break; - --num_outs; - } - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= req.outs_count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if tx is unlocked, add output to result_outs - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - else - { - // while we still need more mixins - while (res.outs.size() < req.outs_count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - - if (res.outs.size() < req.outs_count) - return false; -#if 0 - // if we do not have enough RCT inputs, we can pick from the non RCT ones - // which will have a zero mask - if (res.outs.size() < req.outs_count) - { - LOG_PRINT_L0("Out of RCT inputs (" << res.outs.size() << "/" << req.outs_count << "), using regular ones"); - - // TODO: arbitrary selection, needs better - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req2 = AUTO_VAL_INIT(req2); - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response res2 = AUTO_VAL_INIT(res2); - req2.outs_count = req.outs_count - res.outs.size(); - static const uint64_t amounts[] = {1, 10, 20, 50, 100, 200, 500, 1000, 10000}; - for (uint64_t a: amounts) - req2.amounts.push_back(a); - if (!get_random_outs_for_amounts(req2, res2)) - return false; - - // pick random ones from there - while (res.outs.size() < req.outs_count) - { - int list_idx = rand() % (sizeof(amounts)/sizeof(amounts[0])); - if (!res2.outs[list_idx].outs.empty()) - { - const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry oe = res2.outs[list_idx].outs.back(); - res2.outs[list_idx].outs.pop_back(); - add_out_to_get_rct_random_outs(res.outs, res2.outs[list_idx].amount, oe.global_amount_index); - } - } - } -#endif - - return true; -} -//------------------------------------------------------------------ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { LOG_PRINT_L3("Blockchain::" << __func__); @@ -2294,7 +2082,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons total_height = get_current_blockchain_height(); size_t count = 0, size = 0; blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); - for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) + for(uint64_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); blocks.back().first.first = m_db->get_block_blob_from_height(i); @@ -2517,7 +2305,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh if(m_show_time_stats) { size_t ring_size = !tx.vin.empty() && tx.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(tx.vin[0]).key_offsets.size() : 0; - MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx)); + MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx) << " W: " << get_transaction_weight(tx)); } if (!res) return false; @@ -2574,12 +2362,27 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context // from v8, allow bulletproofs if (hf_version < 8) { - const bool bulletproof = tx.rct_signatures.type == rct::RCTTypeFullBulletproof || tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof; - if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) - { - MERROR("Bulletproofs are not allowed before v8"); - tvc.m_invalid_output = true; - return false; + if (tx.version >= 2) { + const bool bulletproof = rct::is_rct_bulletproof(tx.rct_signatures.type); + if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) + { + MERROR("Bulletproofs are not allowed before v8"); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v9, forbid borromean range proofs + if (hf_version > 8) { + if (tx.version >= 2) { + const bool borromean = rct::is_rct_borromean(tx.rct_signatures.type); + if (borromean) + { + MERROR("Borromean range proofs are not allowed after v8"); + tvc.m_invalid_output = true; + return false; + } } } @@ -2608,7 +2411,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr rv.message = rct::hash2rct(tx_prefix_hash); // mixRing - full and simple store it in opposite ways - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys[0].size()); @@ -2623,7 +2426,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2642,14 +2445,14 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } // II - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { rv.p.MGs.resize(1); rv.p.MGs[0].II.resize(tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2691,7 +2494,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { size_t n_unmixable = 0, n_mixable = 0; size_t mixin = std::numeric_limits<size_t>::max(); - const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; + const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_10 ? 10 : hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; for (const auto& txin : tx.vin) { // non txin_to_key inputs will be rejected below @@ -2720,6 +2523,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } + if (hf_version >= HF_VERSION_MIN_MIXIN_10 && mixin != 10) + { + MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); + tvc.m_low_mixin = true; + return false; + } + if (mixin < min_mixin) { if (n_unmixable == 0) @@ -2915,7 +2725,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, return false; } case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -2965,7 +2775,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (!rct::verRctSimple(rv, false)) + if (!rct::verRctNonSemanticsSimple(rv)) { MERROR_VER("Failed to check ringct signatures!"); return false; @@ -2973,7 +2783,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, break; } case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -3036,6 +2845,22 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, MERROR_VER("Unsupported rct type: " << rv.type); return false; } + + // for bulletproofs, check they're only multi-output after v8 + if (rct::is_rct_bulletproof(rv.type)) + { + if (hf_version < 8) + { + for (const rct::Bulletproof &proof: rv.p.bulletproofs) + { + if (proof.V.size() > 1) + { + MERROR_VER("Multi output bulletproofs are invalid before v8"); + return false; + } + } + } + } } return true; } @@ -3055,7 +2880,7 @@ void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const } //------------------------------------------------------------------ -static uint64_t get_fee_quantization_mask() +uint64_t Blockchain::get_fee_quantization_mask() { static uint64_t mask = 0; if (mask == 0) @@ -3068,16 +2893,27 @@ static uint64_t get_fee_quantization_mask() } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version) +uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version) { - const uint64_t min_block_size = get_min_block_size(version); - const uint64_t fee_per_kb_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; + const uint64_t min_block_weight = get_min_block_weight(version); + if (median_block_weight < min_block_weight) + median_block_weight = min_block_weight; + uint64_t hi, lo; - if (median_block_size < min_block_size) - median_block_size = min_block_size; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + lo = mul128(block_reward, DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT, &hi); + div128_32(hi, lo, min_block_weight, &hi, &lo); + div128_32(hi, lo, median_block_weight, &hi, &lo); + assert(hi == 0); + lo /= 5; + return lo; + } - uint64_t unscaled_fee_per_kb = (fee_per_kb_base * min_block_size / median_block_size); - uint64_t hi, lo = mul128(unscaled_fee_per_kb, block_reward, &hi); + const uint64_t fee_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; + + uint64_t unscaled_fee_base = (fee_base * min_block_weight / median_block_weight); + lo = mul128(unscaled_fee_base, block_reward, &hi); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD % 1000000 == 0, "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD must be divisible by 1000000"); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000 <= std::numeric_limits<uint32_t>::max(), "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD is too large"); @@ -3095,29 +2931,48 @@ uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median } //------------------------------------------------------------------ -bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const +bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); - uint64_t fee_per_kb; - if (version < HF_VERSION_DYNAMIC_FEE) - { - fee_per_kb = FEE_PER_KB; - } - else + uint64_t median = 0; + uint64_t already_generated_coins = 0; + uint64_t base_reward = 0; + if (version >= HF_VERSION_DYNAMIC_FEE) { - uint64_t median = m_current_block_cumul_sz_limit / 2; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - uint64_t base_reward; + median = m_current_block_cumul_weight_limit / 2; + already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; - fee_per_kb = get_dynamic_per_kb_fee(base_reward, median, version); } - MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); - uint64_t needed_fee = blob_size / 1024; - needed_fee += (blob_size % 1024) ? 1 : 0; - needed_fee *= fee_per_kb; + uint64_t needed_fee; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, median, version); + MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); + needed_fee = tx_weight * fee_per_byte; + // quantize fee up to 8 decimals + const uint64_t mask = get_fee_quantization_mask(); + needed_fee = (needed_fee + mask - 1) / mask * mask; + } + else + { + uint64_t fee_per_kb; + if (version < HF_VERSION_DYNAMIC_FEE) + { + fee_per_kb = FEE_PER_KB; + } + else + { + fee_per_kb = get_dynamic_base_fee(base_reward, median, version); + } + MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); + + needed_fee = tx_weight / 1024; + needed_fee += (tx_weight % 1024) ? 1 : 0; + needed_fee *= fee_per_kb; + } if (fee < needed_fee - needed_fee / 50) // keep a little 2% buffer on acceptance - no integer overflow { @@ -3128,7 +2983,7 @@ bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const +uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const { const uint8_t version = get_current_hard_fork_version(); @@ -3138,16 +2993,16 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons if (grace_blocks >= CRYPTONOTE_REWARD_BLOCKS_WINDOW) grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1; - const uint64_t min_block_size = get_min_block_size(version); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); - sz.reserve(grace_blocks); + const uint64_t min_block_weight = get_min_block_weight(version); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + weights.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) - sz.push_back(min_block_size); + weights.push_back(min_block_weight); - uint64_t median = epee::misc_utils::median(sz); - if(median <= min_block_size) - median = min_block_size; + uint64_t median = epee::misc_utils::median(weights); + if(median <= min_block_weight) + median = min_block_weight; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; uint64_t base_reward; @@ -3157,8 +3012,9 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_per_kb_fee(base_reward, median, version); - MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/kB"); + uint64_t fee = get_dynamic_base_fee(base_reward, median, version); + const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; + MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; } @@ -3334,11 +3190,11 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) for (const auto &txid: txids) { cryptonote::transaction tx; - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -3498,8 +3354,8 @@ leave: goto leave; } - size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); - size_t cumulative_block_size = coinbase_blob_size; + size_t coinbase_weight = get_transaction_weight(bl.miner_tx); + size_t cumulative_block_weight = coinbase_weight; std::vector<transaction> txs; key_images_container keys; @@ -3521,7 +3377,7 @@ leave: for (const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; - size_t blob_size = 0; + size_t tx_weight = 0; uint64_t fee = 0; bool relayed = false, do_not_relay = false, double_spend_seen = false; TIME_MEASURE_START(aa); @@ -3540,7 +3396,7 @@ leave: TIME_MEASURE_START(bb); // get transaction with hash <tx_id> from tx_pool - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(!m_tx_pool.take_tx(tx_id, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; @@ -3611,7 +3467,7 @@ leave: TIME_MEASURE_FINISH(cc); t_checktx += cc; fee_summary += fee; - cumulative_block_size += blob_size; + cumulative_block_weight += tx_weight; } m_blocks_txs_check.clear(); @@ -3619,7 +3475,7 @@ leave: TIME_MEASURE_START(vmt); uint64_t base_reward = 0; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) + if(!validate_miner_transaction(bl, cumulative_block_weight, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) { MERROR_VER("Block with id: " << id << " has incorrect miner transaction"); bvc.m_verifivation_failed = true; @@ -3628,11 +3484,11 @@ leave: } TIME_MEASURE_FINISH(vmt); - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; // populate various metadata about the block to be stored alongside it. - block_size = cumulative_block_size; + block_weight = cumulative_block_weight; cumulative_difficulty = current_diffic; // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins @@ -3653,7 +3509,7 @@ leave: { try { - new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + new_height = m_db->add_block(bl, block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { @@ -3678,14 +3534,14 @@ leave: TIME_MEASURE_FINISH(addblock); - // do this after updating the hard fork state since the size limit may change due to fork - update_next_cumulative_size_limit(); + // do this after updating the hard fork state since the weight limit may change due to fork + update_next_cumulative_weight_limit(); - MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); + MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); if(m_show_time_stats) { - MINFO("Height: " << new_height << " blob: " << coinbase_blob_size << " cumm: " - << cumulative_block_size << " p/t: " << block_processing_time << " (" + MINFO("Height: " << new_height << " coinbase weight: " << coinbase_weight << " cumm: " + << cumulative_block_weight << " p/t: " << block_processing_time << " (" << target_calculating_time << "/" << longhash_calculating_time << "/" << t1 << "/" << t2 << "/" << t3 << "/" << t_exists << "/" << t_pool << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock << ")ms"); @@ -3696,24 +3552,26 @@ leave: // appears to be a NOP *and* is called elsewhere. wat? m_tx_pool.on_blockchain_inc(new_height, id); + get_difficulty_for_next_block(); // just to cache it + invalidate_block_template_cache(); return true; } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_size_limit() +bool Blockchain::update_next_cumulative_weight_limit() { - uint64_t full_reward_zone = get_min_block_size(get_current_hard_fork_version()); + uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(sz); - m_current_block_cumul_sz_median = median; + uint64_t median = epee::misc_utils::median(weights); + m_current_block_cumul_weight_median = median; if(median <= full_reward_zone) median = full_reward_zone; - m_current_block_cumul_sz_limit = median*2; + m_current_block_cumul_weight_limit = median*2; return true; } //------------------------------------------------------------------ @@ -3876,11 +3734,13 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) store_blockchain(); m_sync_counter = 0; } - else if (m_db_blocks_per_sync && m_sync_counter >= m_db_blocks_per_sync) + else if (m_db_sync_threshold && ((m_db_sync_on_blocks && m_sync_counter >= m_db_sync_threshold) || (!m_db_sync_on_blocks && m_bytes_to_sync >= m_db_sync_threshold))) { + MDEBUG("Sync threshold met, syncing"); if(m_db_sync_mode == db_async) { m_sync_counter = 0; + m_bytes_to_sync = 0; m_async_service.dispatch(boost::bind(&Blockchain::store_blockchain, this)); } else if(m_db_sync_mode == db_sync) @@ -4041,6 +3901,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete TIME_MEASURE_START(prepare); bool stop_batch; uint64_t bytes = 0; + size_t total_txs = 0; // Order of locking must be: // m_incoming_tx_lock (optional) @@ -4069,7 +3930,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete { bytes += tx_blob.size(); } + total_txs += entry.txs.size(); } + m_bytes_to_sync += bytes; while (!(stop_batch = m_db->batch_start(blocks_entry.size(), bytes))) { m_blockchain_lock.unlock(); m_tx_pool.unlock(); @@ -4128,7 +3991,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete break; } - blocks[i].push_back(block); + blocks[i].push_back(std::move(block)); std::advance(it, 1); } } @@ -4149,7 +4012,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete break; } - blocks[i].push_back(block); + blocks[i].push_back(std::move(block)); std::advance(it, 1); } @@ -4205,6 +4068,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete std::map<uint64_t, std::vector<uint64_t>> offset_map; // [output] stores all output_data_t for each absolute_offset std::map<uint64_t, std::vector<output_data_t>> tx_map; + std::vector<std::pair<cryptonote::transaction, crypto::hash>> txes(total_txs); #define SCAN_TABLE_QUIT(m) \ do { \ @@ -4214,6 +4078,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete } while(0); \ // generate sorted tables for all amounts and absolute offsets + size_t tx_index = 0; for (const auto &entry : blocks_entry) { if (m_cancel) @@ -4221,12 +4086,15 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (const auto &tx_blob : entry.txs) { - crypto::hash tx_hash = null_hash; - crypto::hash tx_prefix_hash = null_hash; - transaction tx; + if (tx_index >= txes.size()) + SCAN_TABLE_QUIT("tx_index is out of sync"); + transaction &tx = txes[tx_index].first; + crypto::hash &tx_prefix_hash = txes[tx_index].second; + ++tx_index; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) + if (!parse_and_validate_tx_base_from_blob(tx_blob, tx)) SCAN_TABLE_QUIT("Could not parse tx from incoming blocks."); + cryptonote::get_transaction_prefix_hash(tx, tx_prefix_hash); auto its = m_scan_table.find(tx_prefix_hash); if (its != m_scan_table.end()) @@ -4312,9 +4180,8 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete } } - int total_txs = 0; - // now generate a table for each tx_prefix and k_image hashes + tx_index = 0; for (const auto &entry : blocks_entry) { if (m_cancel) @@ -4322,14 +4189,12 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (const auto &tx_blob : entry.txs) { - crypto::hash tx_hash = null_hash; - crypto::hash tx_prefix_hash = null_hash; - transaction tx; + if (tx_index >= txes.size()) + SCAN_TABLE_QUIT("tx_index is out of sync"); + const transaction &tx = txes[tx_index].first; + const crypto::hash &tx_prefix_hash = txes[tx_index].second; + ++tx_index; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) - SCAN_TABLE_QUIT("Could not parse tx from incoming blocks."); - - ++total_txs; auto its = m_scan_table.find(tx_prefix_hash); if (its == m_scan_table.end()) SCAN_TABLE_QUIT("Tx not found on scan table from incoming blocks."); @@ -4418,7 +4283,7 @@ bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, con return m_db->for_all_txpool_txes(f, include_blob, include_unrelayed_txes); } -void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync) +void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync) { if (sync_mode == db_defaultsync) { @@ -4427,7 +4292,8 @@ void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, } m_db_sync_mode = sync_mode; m_fast_sync = fast_sync; - m_db_blocks_per_sync = blocks_per_sync; + m_db_sync_on_blocks = sync_on_blocks; + m_db_sync_threshold = sync_threshold; m_max_prepare_blocks_threads = maxthreads; } @@ -4602,14 +4468,14 @@ void Blockchain::load_compiled_in_block_hashes() std::vector<transaction> txs; m_tx_pool.get_transactions(txs); - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; transaction pool_tx; for(const transaction &tx : txs) { crypto::hash tx_hash = get_transaction_hash(tx); - m_tx_pool.take_tx(tx_hash, pool_tx, blob_size, fee, relayed, do_not_relay, double_spend_seen); + m_tx_pool.take_tx(tx_hash, pool_tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen); } } } @@ -4661,6 +4527,24 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t he return m_db->for_all_outputs(amount, f);; } +void Blockchain::invalidate_block_template_cache() +{ + MDEBUG("Invalidating block template cache"); + m_btc_valid = false; +} + +void Blockchain::cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t expected_reward, uint64_t pool_cookie) +{ + MDEBUG("Setting block template cache"); + m_btc = b; + m_btc_address = address; + m_btc_nonce = nonce; + m_btc_difficulty = diff; + m_btc_expected_reward = expected_reward; + m_btc_pool_cookie = pool_cookie; + m_btc_valid = true; +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index d95c8ed15..50ceccd0f 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -95,7 +95,7 @@ namespace cryptonote { block bl; //!< the block uint64_t height; //!< the height of the block in the blockchain - size_t block_cumulative_size; //!< the size (in bytes) of the block + size_t block_cumulative_weight; //!< the weight of the block difficulty_type cumulative_difficulty; //!< the accumulated difficulty after that block uint64_t already_generated_coins; //!< the total coins minted after that block }; @@ -447,16 +447,6 @@ namespace cryptonote uint64_t get_num_mature_outputs(uint64_t amount) const; /** - * @brief get random outputs (indices) for an amount - * - * @param amount the amount - * @param count the number of random outputs to choose - * - * @return the outputs' amount-global indices - */ - std::vector<uint64_t> get_random_outputs(uint64_t amount, uint64_t count) const; - - /** * @brief get the public key for an output * * @param amount the output amount @@ -467,22 +457,6 @@ namespace cryptonote crypto::public_key get_output_key(uint64_t amount, uint64_t global_index) const; /** - * @brief gets random outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @brief gets specific outputs to mix with * * This function takes an RPC request for outputs to mix with @@ -509,23 +483,6 @@ namespace cryptonote void get_output_key_mask_unlocked(const uint64_t& amount, const uint64_t& index, crypto::public_key& key, rct::key& mask, bool& unlocked) const; /** - * @brief gets random ringct outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices - * and the matching keys. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @brief gets per block distribution of outputs of a given amount * * @param amount the amount to get a ditribution for @@ -579,46 +536,57 @@ namespace cryptonote bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false); /** - * @brief get dynamic per kB fee for a given block size + * @brief get fee quantization mask + * + * The dynamic fee may be quantized, to mask out the last decimal places + * + * @return the fee quantized mask + */ + static uint64_t get_fee_quantization_mask(); + + /** + * @brief get dynamic per kB or byte fee for a given block weight * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. * * @param block_reward the current block reward - * @param median_block_size the median blob's size in the past window + * @param median_block_weight the median block weight in the past window * @param version hard fork version for rules and constants to use * - * @return the per kB fee + * @return the fee */ - static uint64_t get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version); + static uint64_t get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version); /** - * @brief get dynamic per kB fee estimate for the next few blocks + * @brief get dynamic per kB or byte fee estimate for the next few blocks * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. This function - * calculates an estimate for a dynamic fee which will be valid for - * the next grace_blocks + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. + * This function calculates an estimate for a dynamic fee which will be + * valid for the next grace_blocks * * @param grace_blocks number of blocks we want the fee to be valid for * - * @return the per kB fee estimate + * @return the fee estimate */ - uint64_t get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const; + uint64_t get_dynamic_base_fee_estimate(uint64_t grace_blocks) const; /** * @brief validate a transaction's fee * * This function validates the fee is enough for the transaction. - * This is based on the size of the transaction blob, and, after a - * height threshold, on the average size of transaction in a past window + * This is based on the weight of the transaction, and, after a + * height threshold, on the average weight of transaction in a past window * - * @param blob_size the transaction blob's size + * @param tx_weight the transaction weight * @param fee the fee * * @return true if the fee is enough, false otherwise */ - bool check_fee(size_t blob_size, uint64_t fee) const; + bool check_fee(size_t tx_weight, uint64_t fee) const; /** * @brief check that a transaction's outputs conform to current standards @@ -635,18 +603,18 @@ namespace cryptonote bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc); /** - * @brief gets the blocksize limit based on recent blocks + * @brief gets the block weight limit based on recent blocks * * @return the limit */ - uint64_t get_current_cumulative_blocksize_limit() const; + uint64_t get_current_cumulative_block_weight_limit() const; /** - * @brief gets the blocksize median based on recent blocks (same window as for the limit) + * @brief gets the block weight median based on recent blocks (same window as for the limit) * * @return the median */ - uint64_t get_current_cumulative_blocksize_median() const; + uint64_t get_current_cumulative_block_weight_median() const; /** * @brief gets the difficulty of the block with a given height @@ -729,11 +697,12 @@ namespace cryptonote * @brief sets various performance options * * @param maxthreads max number of threads when preparing blocks for addition - * @param blocks_per_sync number of blocks to cache before syncing to database + * @param sync_on_blocks whether to sync based on blocks or bytes + * @param sync_threshold number of blocks/bytes to cache before syncing to database * @param sync_mode the ::blockchain_db_sync_mode to use * @param fast_sync sync using built-in block hashes as trusted */ - void set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, + void set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync); /** @@ -1000,8 +969,8 @@ namespace cryptonote // main chain transactions_container m_transactions; - size_t m_current_block_cumul_sz_limit; - size_t m_current_block_cumul_sz_median; + size_t m_current_block_cumul_weight_limit; + size_t m_current_block_cumul_weight_median; // metadata containers std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table; @@ -1017,11 +986,13 @@ namespace cryptonote bool m_fast_sync; bool m_show_time_stats; bool m_db_default_sync; - uint64_t m_db_blocks_per_sync; + bool m_db_sync_on_blocks; + uint64_t m_db_sync_threshold; uint64_t m_max_prepare_blocks_threads; uint64_t m_fake_pow_calc_time; uint64_t m_fake_scan_time; uint64_t m_sync_counter; + uint64_t m_bytes_to_sync; std::vector<uint64_t> m_timestamps; std::vector<difficulty_type> m_difficulties; uint64_t m_timestamps_and_difficulties_height; @@ -1052,6 +1023,15 @@ namespace cryptonote std::atomic<bool> m_cancel; + // block template cache + block m_btc; + account_public_address m_btc_address; + blobdata m_btc_nonce; + difficulty_type m_btc_difficulty; + uint64_t m_btc_pool_cookie; + uint64_t m_btc_expected_reward; + bool m_btc_valid; + /** * @brief collects the keys for all outputs being "spent" as an input * @@ -1213,7 +1193,7 @@ namespace cryptonote * and that his miner transaction totals reward + fee. * * @param b the block containing the miner transaction to be validated - * @param cumulative_block_size the block's size + * @param cumulative_block_weight the block's weight * @param fee the total fees collected in the block * @param base_reward return-by-reference the new block's generated coins * @param already_generated_coins the amount of currency generated prior to this block @@ -1222,7 +1202,7 @@ namespace cryptonote * * @return false if anything is found wrong with the miner transaction, otherwise true */ - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); + bool validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); /** * @brief reverts the blockchain to its previous state following a failed switch @@ -1239,32 +1219,14 @@ namespace cryptonote bool rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height); /** - * @brief gets recent block sizes for median calculation - * - * get the block sizes of the last <count> blocks, and return by reference <sz>. + * @brief gets recent block weights for median calculation * - * @param sz return-by-reference the list of sizes - * @param count the number of blocks to get sizes for - */ - void get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; - - /** - * @brief adds the given output to the requested set of random outputs + * get the block weights of the last <count> blocks, and return by reference <sz>. * - * @param result_outs return-by-reference the set the output is to be added to - * @param amount the output amount - * @param i the output index (indexed to amount) + * @param sz return-by-reference the list of weights + * @param count the number of blocks to get weights for */ - void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; - - /** - * @brief adds the given output to the requested set of random ringct outputs - * - * @param outs return-by-reference the set the output is to be added to - * @param amount the output amount (0 for rct inputs) - * @param i the rct output index - */ - void add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const; + void get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) @@ -1361,11 +1323,11 @@ namespace cryptonote bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); /** - * @brief calculate the block size limit for the next block to be added + * @brief calculate the block weight limit for the next block to be added * * @return true */ - bool update_next_cumulative_size_limit(); + bool update_next_cumulative_weight_limit(); void return_tx_to_pool(std::vector<transaction> &txs); /** @@ -1407,5 +1369,17 @@ namespace cryptonote * that implicit data. */ bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys); + + /** + * @brief invalidates any cached block template + */ + void invalidate_block_template_cache(); + + /** + * @brief stores a new cached block template + * + * At some point, may be used to push an update to miners + */ + void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t expected_reward, uint64_t pool_cookie); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 18490c65e..7cbf414b7 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -162,10 +162,10 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; - static const command_line::arg_descriptor<size_t> arg_max_txpool_size = { - "max-txpool-size" - , "Set maximum txpool size in bytes." - , DEFAULT_TXPOOL_MAX_SIZE + static const command_line::arg_descriptor<size_t> arg_max_txpool_weight = { + "max-txpool-weight" + , "Set maximum txpool weight in bytes." + , DEFAULT_TXPOOL_MAX_WEIGHT }; //----------------------------------------------------------------------------------------------- @@ -274,7 +274,7 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); - command_line::add_arg(desc, arg_max_txpool_size); + command_line::add_arg(desc, arg_max_txpool_weight); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -402,7 +402,7 @@ namespace cryptonote bool fast_sync = command_line::get_arg(vm, arg_fast_block_sync) != 0; uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); - size_t max_txpool_size = command_line::get_arg(vm, arg_max_txpool_size); + size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -439,9 +439,10 @@ namespace cryptonote MGINFO("Loading blockchain from folder " << folder.string() << " ..."); const std::string filename = folder.string(); - // default to fast:async:1 + // default to fast:async:1 if overridden blockchain_db_sync_mode sync_mode = db_defaultsync; - uint64_t blocks_per_sync = 1; + bool sync_on_blocks = true; + uint64_t sync_threshold = 1; if (m_nettype == FAKECHAIN) { @@ -491,7 +492,7 @@ namespace cryptonote else if(options[0] == "fastest") { db_flags = DBF_FASTEST; - blocks_per_sync = 1000; // default to fastest:async:1000 + sync_threshold = 1000; // default to fastest:async:1000 sync_mode = db_sync_mode_is_default ? db_defaultsync : db_async; } else @@ -509,9 +510,22 @@ namespace cryptonote if(options.size() >= 3 && !safemode) { char *endptr; - uint64_t bps = strtoull(options[2].c_str(), &endptr, 0); - if (*endptr == '\0') - blocks_per_sync = bps; + uint64_t threshold = strtoull(options[2].c_str(), &endptr, 0); + if (*endptr == '\0' || !strcmp(endptr, "blocks")) + { + sync_on_blocks = true; + sync_threshold = threshold; + } + else if (!strcmp(endptr, "bytes")) + { + sync_on_blocks = false; + sync_threshold = threshold; + } + else + { + LOG_ERROR("Invalid db sync mode: " << options[2]); + return false; + } } if (db_salvage) @@ -528,7 +542,7 @@ namespace cryptonote } m_blockchain_storage.set_user_options(blocks_threads, - blocks_per_sync, sync_mode, fast_sync); + sync_on_blocks, sync_threshold, sync_mode, fast_sync); const std::pair<uint8_t, uint64_t> regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; const cryptonote::test_options regtest_test_options = { @@ -537,7 +551,7 @@ namespace cryptonote const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty); - r = m_mempool.init(max_txpool_size); + r = m_mempool.init(max_txpool_weight); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); // now that we have a valid m_blockchain_storage, we can clean out any @@ -678,26 +692,123 @@ namespace cryptonote return false; } + return true; + } + //----------------------------------------------------------------------------------------------- + void core::set_semantics_failed(const crypto::hash &tx_hash) + { + LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + bad_semantics_txes_lock.lock(); + bad_semantics_txes[0].insert(tx_hash); + if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + { + std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); + bad_semantics_txes[0].clear(); + } + bad_semantics_txes_lock.unlock(); + } + //----------------------------------------------------------------------------------------------- + static bool is_canonical_bulletproof_layout(const std::vector<rct::Bulletproof> &proofs) + { + if (proofs.size() != 1) + return false; + const size_t sz = proofs[0].V.size(); + if (sz == 0 || sz > BULLETPROOF_MAX_OUTPUTS) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block) + { + bool ret = true; if (keeped_by_block && get_blockchain_storage().is_within_compiled_block_hash_area()) { MTRACE("Skipping semantics check for tx kept by block in embedded hash area"); + return true; } - else if(!check_tx_semantic(tx, keeped_by_block)) + + std::vector<const rct::rctSig*> rvv; + for (size_t n = 0; n < tx_info.size(); ++n) { - LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); - tvc.m_verifivation_failed = true; - bad_semantics_txes_lock.lock(); - bad_semantics_txes[0].insert(tx_hash); - if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + if (!check_tx_semantic(*tx_info[n].tx, keeped_by_block)) { - std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); - bad_semantics_txes[0].clear(); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + continue; + } + + if (tx_info[n].tx->version < 2) + continue; + const rct::rctSig &rv = tx_info[n].tx->rct_signatures; + switch (rv.type) { + case rct::RCTTypeNull: + // coinbase should not come here, so we reject for all other types + MERROR_VER("Unexpected Null rctSig type"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + case rct::RCTTypeSimple: + if (!rct::verRctSemanticsSimple(rv)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeFull: + if (!rct::verRct(rv, true)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeBulletproof: + if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) + { + MERROR_VER("Bulletproof does not have canonical form"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + rvv.push_back(&rv); // delayed batch verification + break; + default: + MERROR_VER("Unknown rct type: " << rv.type); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + } + if (!rvv.empty() && !rct::verRctSemanticsSimple(rvv)) + { + LOG_PRINT_L1("One transaction among this group has bad semantics, verifying one at a time"); + ret = false; + const bool assumed_bad = rvv.size() == 1; // if there's only one tx, it must be the bad one + for (size_t n = 0; n < tx_info.size(); ++n) + { + if (!tx_info[n].result) + continue; + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof) + continue; + if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) + { + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + } } - bad_semantics_txes_lock.unlock(); - return false; } - return true; + return ret; } //----------------------------------------------------------------------------------------------- bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) @@ -755,6 +866,16 @@ namespace cryptonote } waiter.wait(&tpool); + std::vector<tx_verification_batch_info> tx_info; + tx_info.reserve(tx_blobs.size()); + for (size_t i = 0; i < tx_blobs.size(); i++) { + if (!results[i].res) + continue; + tx_info.push_back({&results[i].tx, results[i].hash, tvc[i], results[i].res}); + } + if (!tx_info.empty()) + handle_incoming_tx_accumulated_batch(tx_info, keeped_by_block); + bool ok = true; it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { @@ -764,7 +885,8 @@ namespace cryptonote continue; } - ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, it->size(), tvc[i], keeped_by_block, relayed, do_not_relay); + const size_t weight = get_transaction_weight(results[i].tx, it->size()); + ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -847,9 +969,9 @@ namespace cryptonote } // for version > 1, ringct signatures check verifies amounts match - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_transaction_weight(tx) >= m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - MERROR_VER("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + MERROR_VER("tx is too large " << get_transaction_weight(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -872,36 +994,6 @@ namespace cryptonote return false; } - if (tx.version >= 2) - { - const rct::rctSig &rv = tx.rct_signatures; - switch (rv.type) { - case rct::RCTTypeNull: - // coinbase should not come here, so we reject for all other types - MERROR_VER("Unexpected Null rctSig type"); - return false; - case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: - if (!rct::verRctSimple(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: - if (!rct::verRct(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - default: - MERROR_VER("Unknown rct type: " << rv.type); - return false; - } - } - return true; } //----------------------------------------------------------------------------------------------- @@ -1011,7 +1103,8 @@ namespace cryptonote crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); blobdata bl; t_serializable_object_to_blob(tx, bl); - return add_new_tx(tx, tx_hash, tx_prefix_hash, bl.size(), tvc, keeped_by_block, relayed, do_not_relay); + size_t tx_weight = get_transaction_weight(tx, bl.size()); + return add_new_tx(tx, tx_hash, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -1019,7 +1112,7 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if (keeped_by_block) get_blockchain_storage().on_new_tx_from_block(tx); @@ -1037,7 +1130,7 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() @@ -1088,21 +1181,11 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- - bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const - { - return m_blockchain_storage.get_random_outs_for_amounts(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { return m_blockchain_storage.get_outs(req, res); } //----------------------------------------------------------------------------------------------- - bool core::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const - { - return m_blockchain_storage.get_random_rct_outs(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const { return m_blockchain_storage.get_output_distribution(amount, from_height, to_height, start_height, distribution, base); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 84e1bb918..b40575ae9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -551,13 +551,6 @@ namespace cryptonote difficulty_type get_block_cumulative_difficulty(uint64_t height) const; /** - * @copydoc Blockchain::get_random_outs_for_amounts - * - * @note see Blockchain::get_random_outs_for_amounts - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @copydoc Blockchain::get_outs * * @note see Blockchain::get_outs @@ -565,14 +558,6 @@ namespace cryptonote bool get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const; /** - * - * @copydoc Blockchain::get_random_rct_outs - * - * @note see Blockchain::get_random_rct_outs - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @copydoc Blockchain::get_output_distribution * * @brief get per block distribution of outputs of a given amount @@ -788,12 +773,12 @@ namespace cryptonote * * @param tx_hash the transaction's hash * @param tx_prefix_hash the transaction prefix' hash - * @param blob_size the size of the transaction + * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -864,9 +849,12 @@ namespace cryptonote * @return true if all the checks pass, otherwise false */ bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; + void set_semantics_failed(const crypto::hash &tx_hash); bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; + bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block); /** * @copydoc miner::on_block_chain_update diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 071ce591e..fb2af9ceb 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -74,7 +74,7 @@ namespace cryptonote LOG_PRINT_L2("destinations include " << num_stdaddresses << " standard addresses and " << num_subaddresses << " subaddresses"); } //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { tx.vin.clear(); tx.vout.clear(); tx.extra.clear(); @@ -89,7 +89,7 @@ namespace cryptonote in.height = height; uint64_t block_reward; - if(!get_block_reward(median_size, current_block_size, already_generated_coins, block_reward, hard_fork_version)) + if(!get_block_reward(median_weight, current_block_weight, already_generated_coins, block_reward, hard_fork_version)) { LOG_PRINT_L0("Block is too big"); return false; @@ -195,7 +195,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -491,7 +491,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1; + bool use_simple_rct = sources.size() > 1 || range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -516,6 +516,7 @@ namespace cryptonote uint64_t amount_in = 0, amount_out = 0; rct::ctkeyV inSk; + inSk.reserve(sources.size()); // mixRing indexing is done the other way round for simple rct::ctkeyM mixRing(use_simple_rct ? sources.size() : n_total_outs); rct::keyV destinations; @@ -532,6 +533,7 @@ namespace cryptonote ctkey.dest = rct::sk2rct(in_contexts[i].in_ephemeral.sec); ctkey.mask = sources[i].mask; inSk.push_back(ctkey); + memwipe(&ctkey, sizeof(rct::ctkey)); // inPk: (public key, commitment) // will be done when filling in mixRing if (msout) @@ -587,9 +589,10 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, bulletproof, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, range_proof_type, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, bulletproof, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, hwdev); // same index assumption + memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -601,7 +604,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -619,7 +622,7 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, bulletproof, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, range_proof_type, msout); hwdev.close_tx(); return r; } @@ -631,7 +634,7 @@ namespace cryptonote crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index a5d149fca..f2cf7b6ca 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -37,7 +37,7 @@ namespace cryptonote { //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); struct tx_source_entry { @@ -90,8 +90,8 @@ namespace cryptonote //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL); bool generate_genesis_block( block& bl diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index eac0f1f57..a725eac6e 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -80,9 +80,13 @@ namespace cryptonote return amount * ACCEPT_THRESHOLD; } - uint64_t get_transaction_size_limit(uint8_t version) + uint64_t get_transaction_weight_limit(uint8_t version) { - return get_min_block_size(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + // from v8, limit a tx to 50% of the minimum block weight + if (version >= 8) + return get_min_block_weight(version) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return get_min_block_weight(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } // This class is meant to create a batch when none currently exists. @@ -102,12 +106,12 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //--------------------------------------------------------------------------------- - tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_size(DEFAULT_TXPOOL_MAX_SIZE), m_txpool_size(0) + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_cookie(0) { } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -173,17 +177,17 @@ namespace cryptonote fee = tx.rct_signatures.txnFee; } - if (!kept_by_block && !m_blockchain.check_fee(blob_size, fee)) + if (!kept_by_block && !m_blockchain.check_fee(tx_weight, fee)) { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; return false; } - size_t tx_size_limit = get_transaction_size_limit(version); - if (!kept_by_block && blob_size > tx_size_limit) + size_t tx_weight_limit = get_transaction_weight_limit(version); + if ((!kept_by_block || version >= HF_VERSION_PER_BYTE_FEE) && tx_weight > tx_weight_limit) { - LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit); + LOG_PRINT_L1("transaction is too heavy: " << tx_weight << " bytes, maximum weight: " << tx_weight_limit); tvc.m_verifivation_failed = true; tvc.m_too_big = true; return false; @@ -227,7 +231,7 @@ namespace cryptonote // may become valid again, so ignore the failed inputs check. if(kept_by_block) { - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.fee = fee; meta.max_used_block_id = null_hash; meta.max_used_block_height = 0; @@ -248,7 +252,7 @@ namespace cryptonote m_blockchain.add_txpool_tx(tx, meta); if (!insert_key_images(tx, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -267,7 +271,7 @@ namespace cryptonote }else { //update transactions container - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.kept_by_block = kept_by_block; meta.fee = fee; meta.max_used_block_id = max_used_block_id; @@ -290,7 +294,7 @@ namespace cryptonote m_blockchain.add_txpool_tx(tx, meta); if (!insert_key_images(tx, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -304,11 +308,13 @@ namespace cryptonote } tvc.m_verifivation_failed = false; - m_txpool_size += blob_size; + m_txpool_weight += tx_weight; + + ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size)); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)tx_weight)); - prune(m_txpool_max_size); + prune(m_txpool_max_weight); return true; } @@ -319,34 +325,35 @@ namespace cryptonote size_t blob_size = 0; if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0) return false; - return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, get_transaction_weight(tx, blob_size), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- - size_t tx_memory_pool::get_txpool_size() const + size_t tx_memory_pool::get_txpool_weight() const { CRITICAL_REGION_LOCAL(m_transactions_lock); - return m_txpool_size; + return m_txpool_weight; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_txpool_max_size(size_t bytes) + void tx_memory_pool::set_txpool_max_weight(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); - m_txpool_max_size = bytes; + m_txpool_max_weight = bytes; } //--------------------------------------------------------------------------------- void tx_memory_pool::prune(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); if (bytes == 0) - bytes = m_txpool_max_size; + bytes = m_txpool_max_weight; CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); + bool changed = false; // this will never remove the first one, but we don't care auto it = --m_txs_by_fee_and_receive_time.end(); while (it != m_txs_by_fee_and_receive_time.begin()) { - if (m_txpool_size <= bytes) + if (m_txpool_weight <= bytes) break; try { @@ -371,12 +378,13 @@ namespace cryptonote return; } // remove first, in case this throws, so key images aren't removed - MINFO("Pruning tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruning tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); + m_txpool_weight -= it->first.second; remove_transaction_keyimages(tx); - MINFO("Pruned tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruned tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); m_txs_by_fee_and_receive_time.erase(it--); + changed = true; } catch (const std::exception &e) { @@ -384,8 +392,10 @@ namespace cryptonote return; } } - if (m_txpool_size > bytes) - MINFO("Pool size after pruning is larger than limit: " << m_txpool_size << "/" << bytes); + if (changed) + ++m_cookie; + if (m_txpool_weight > bytes) + MINFO("Pool weight after pruning is larger than limit: " << m_txpool_weight << "/" << bytes); } //--------------------------------------------------------------------------------- bool tx_memory_pool::insert_key_images(const transaction &tx, bool kept_by_block) @@ -401,6 +411,7 @@ namespace cryptonote auto ins_res = kei_image_set.insert(id); CHECK_AND_ASSERT_MES(ins_res.second, false, "internal error: try to insert duplicate iterator in key_image set"); } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- @@ -435,10 +446,11 @@ namespace cryptonote } } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) + bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -462,7 +474,7 @@ namespace cryptonote MERROR("Failed to parse tx from txpool"); return false; } - blob_size = meta.blob_size; + tx_weight = meta.weight; fee = meta.fee; relayed = meta.relayed; do_not_relay = meta.do_not_relay; @@ -470,7 +482,7 @@ namespace cryptonote // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); - m_txpool_size -= blob_size; + m_txpool_weight -= tx_weight; remove_transaction_keyimages(tx); } catch (const std::exception &e) @@ -480,6 +492,7 @@ namespace cryptonote } m_txs_by_fee_and_receive_time.erase(sorted_it); + ++m_cookie; return true; } //--------------------------------------------------------------------------------- @@ -543,7 +556,7 @@ namespace cryptonote { // remove first, so we only remove key images if the tx removal succeeds m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= bd.size(); + m_txpool_weight -= get_transaction_weight(tx, bd.size()); remove_transaction_keyimages(tx); } } @@ -553,6 +566,7 @@ namespace cryptonote // ignore error } } + ++m_cookie; } return true; } @@ -660,7 +674,7 @@ namespace cryptonote const uint64_t now = time(NULL); backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); + backlog.push_back({meta.weight, meta.fee, meta.receive_time - now}); return true; }, false, include_unrelayed_txes); } @@ -672,15 +686,15 @@ namespace cryptonote const uint64_t now = time(NULL); std::map<uint64_t, txpool_histo> agebytes; stats.txs_total = m_blockchain.get_txpool_tx_count(include_unrelayed_txes); - std::vector<uint32_t> sizes; - sizes.reserve(stats.txs_total); - m_blockchain.for_all_txpool_txes([&stats, &sizes, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - sizes.push_back(meta.blob_size); - stats.bytes_total += meta.blob_size; - if (!stats.bytes_min || meta.blob_size < stats.bytes_min) - stats.bytes_min = meta.blob_size; - if (meta.blob_size > stats.bytes_max) - stats.bytes_max = meta.blob_size; + std::vector<uint32_t> weights; + weights.reserve(stats.txs_total); + m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + weights.push_back(meta.weight); + stats.bytes_total += meta.weight; + if (!stats.bytes_min || meta.weight < stats.bytes_min) + stats.bytes_min = meta.weight; + if (meta.weight > stats.bytes_max) + stats.bytes_max = meta.weight; if (!meta.relayed) stats.num_not_relayed++; stats.fee_total += meta.fee; @@ -692,12 +706,12 @@ namespace cryptonote stats.num_failing++; uint64_t age = now - meta.receive_time + (now == meta.receive_time); agebytes[age].txs++; - agebytes[age].bytes += meta.blob_size; + agebytes[age].bytes += meta.weight; if (meta.double_spend_seen) ++stats.num_double_spends; return true; }, false, include_unrelayed_txes); - stats.bytes_med = epee::misc_utils::median(sizes); + stats.bytes_med = epee::misc_utils::median(weights); if (stats.txs_total > 1) { /* looking for 98th percentile */ @@ -761,7 +775,8 @@ namespace cryptonote return true; } txi.tx_json = obj_to_json_str(tx); - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -832,7 +847,8 @@ namespace cryptonote return true; } txi.tx = tx; - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -1051,6 +1067,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + bool changed = false; LockedTXN lock(m_blockchain); for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -1071,6 +1088,7 @@ namespace cryptonote { MDEBUG("Marking " << txid << " as double spending " << itk.k_image); meta.double_spend_seen = true; + changed = true; try { m_blockchain.update_txpool_tx(txid, meta); @@ -1084,6 +1102,8 @@ namespace cryptonote } } } + if (changed) + ++m_cookie; } //--------------------------------------------------------------------------------- std::string tx_memory_pool::print_pool(bool short_format) const @@ -1102,7 +1122,8 @@ namespace cryptonote } ss << obj_to_json_str(tx) << std::endl; } - ss << "blob_size: " << meta.blob_size << std::endl + ss << "blob_size: " << (short_format ? "-" : std::to_string(txblob->size())) << std::endl + << "weight: " << meta.weight << std::endl << "fee: " << print_money(meta.fee) << std::endl << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl << "double_spend_seen: " << (meta.double_spend_seen ? 'T' : 'F') << std::endl @@ -1117,30 +1138,30 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version) + bool tx_memory_pool::fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); uint64_t best_coinbase = 0, coinbase = 0; - total_size = 0; + total_weight = 0; fee = 0; //baseline empty block - get_block_reward(median_size, total_size, already_generated_coins, best_coinbase, version); + get_block_reward(median_weight, total_weight, already_generated_coins, best_coinbase, version); - size_t max_total_size_pre_v5 = (130 * median_size) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size_v5 = 2 * median_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size = version >= 5 ? max_total_size_v5 : max_total_size_pre_v5; + size_t max_total_weight_pre_v5 = (130 * median_weight) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight_v5 = 2 * median_weight - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight = version >= 5 ? max_total_weight_v5 : max_total_weight_pre_v5; std::unordered_set<crypto::key_image> k_images; - LOG_PRINT_L2("Filling block template, median size " << median_size << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); + LOG_PRINT_L2("Filling block template, median weight " << median_weight << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); LockedTXN lock(m_blockchain); auto sorted_it = m_txs_by_fee_and_receive_time.begin(); - while (sorted_it != m_txs_by_fee_and_receive_time.end()) + for (; sorted_it != m_txs_by_fee_and_receive_time.end(); ++sorted_it) { txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta)) @@ -1148,13 +1169,12 @@ namespace cryptonote MERROR(" failed to find tx meta"); continue; } - LOG_PRINT_L2("Considering " << sorted_it->second << ", size " << meta.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase)); - // Can not exceed maximum block size - if (max_total_size < total_size + meta.blob_size) + // Can not exceed maximum block weight + if (max_total_weight < total_weight + meta.weight) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } @@ -1164,27 +1184,25 @@ namespace cryptonote // If we're getting lower coinbase tx, // stop including more tx uint64_t block_reward; - if(!get_block_reward(median_size, total_size + meta.blob_size, already_generated_coins, block_reward, version)) + if(!get_block_reward(median_weight, total_weight + meta.weight, already_generated_coins, block_reward, version)) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } coinbase = block_reward + fee + meta.fee; if (coinbase < template_accept_threshold(best_coinbase)) { LOG_PRINT_L2(" would decrease coinbase to " << print_money(coinbase)); - sorted_it++; continue; } } else { - // If we've exceeded the penalty free size, + // If we've exceeded the penalty free weight, // stop including more tx - if (total_size > median_size) + if (total_weight > median_weight) { - LOG_PRINT_L2(" would exceed median block size"); + LOG_PRINT_L2(" would exceed median block weight"); break; } } @@ -1221,28 +1239,25 @@ namespace cryptonote if (!ready) { LOG_PRINT_L2(" not ready to go"); - sorted_it++; continue; } if (have_key_images(k_images, tx)) { LOG_PRINT_L2(" key images already seen"); - sorted_it++; continue; } bl.tx_hashes.push_back(sorted_it->second); - total_size += meta.blob_size; + total_weight += meta.weight; fee += meta.fee; best_coinbase = coinbase; append_key_images(k_images, tx); - sorted_it++; - LOG_PRINT_L2(" added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2(" added, new block weight " << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase)); } expected_reward = best_coinbase; - LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, size " - << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase) + LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, weight " + << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase) << " (including " << print_money(fee) << " in fees)"); return true; } @@ -1251,14 +1266,14 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - size_t tx_size_limit = get_transaction_size_limit(version); + size_t tx_weight_limit = get_transaction_weight_limit(version); std::unordered_set<crypto::hash> remove; - m_txpool_size = 0; - m_blockchain.for_all_txpool_txes([this, &remove, tx_size_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { - m_txpool_size += meta.blob_size; - if (meta.blob_size > tx_size_limit) { - LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.blob_size << " bytes), removing it from pool"); + m_txpool_weight = 0; + m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_txpool_weight += meta.weight; + if (meta.weight > tx_weight_limit) { + LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.weight << " bytes), removing it from pool"); remove.insert(txid); } else if (m_blockchain.have_tx(txid)) { @@ -1285,7 +1300,7 @@ namespace cryptonote } // remove tx from db first m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); + m_txpool_weight -= get_transaction_weight(tx, txblob.size()); remove_transaction_keyimages(tx); auto sorted_it = find_tx_in_sorted_container(txid); if (sorted_it == m_txs_by_fee_and_receive_time.end()) @@ -1305,18 +1320,20 @@ namespace cryptonote } } } + if (n_removed > 0) + ++m_cookie; return n_removed; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::init(size_t max_txpool_size) + bool tx_memory_pool::init(size_t max_txpool_weight) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - m_txpool_max_size = max_txpool_size ? max_txpool_size : DEFAULT_TXPOOL_MAX_SIZE; + m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); m_spent_key_images.clear(); - m_txpool_size = 0; + m_txpool_weight = 0; std::vector<crypto::hash> remove; // first add the not kept by block, then the kept by block, @@ -1338,8 +1355,8 @@ namespace cryptonote MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.blob_size, meta.receive_time), txid); - m_txpool_size += meta.blob_size; + m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid); + m_txpool_weight += meta.weight; return true; }, true); if (!r) @@ -1361,6 +1378,10 @@ namespace cryptonote } } } + + m_cookie = 0; + + // Ignore deserialization error return true; } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 4ade7ddbe..892cadc69 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -84,7 +84,7 @@ namespace cryptonote * * This handling includes: * storing the transactions - * organizing the transactions by fee per size + * organizing the transactions by fee per weight unit * taking/giving transactions to and from various other components * saving the transactions to disk on shutdown * helping create a new block template by choosing transactions for it @@ -105,9 +105,9 @@ namespace cryptonote * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) * * @param id the transaction's hash - * @param blob_size the transaction's size + * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -133,7 +133,7 @@ namespace cryptonote * * @param id the hash of the transaction * @param tx return-by-reference the transaction taken - * @param blob_size return-by-reference the transaction's size + * @param tx_weight return-by-reference the transaction's weight * @param fee the transaction fee * @param relayed return-by-reference was transaction relayed to us by the network? * @param do_not_relay return-by-reference is transaction not to be relayed to the network? @@ -141,7 +141,7 @@ namespace cryptonote * * @return true unless the transaction cannot be found in the pool */ - bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); + bool take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); /** * @brief checks if the pool has a transaction with the given hash @@ -198,11 +198,11 @@ namespace cryptonote /** * @brief loads pool state (if any) from disk, and initializes pool * - * @param max_txpool_size the max size in bytes + * @param max_txpool_weight the max weight in bytes * * @return true */ - bool init(size_t max_txpool_size = 0); + bool init(size_t max_txpool_weight = 0); /** * @brief attempts to save the transaction pool state to disk @@ -219,16 +219,16 @@ namespace cryptonote * @brief Chooses transactions for a block to include * * @param bl return-by-reference the block to fill in with transactions - * @param median_size the current median block size + * @param median_weight the current median block weight * @param already_generated_coins the current total number of coins "minted" - * @param total_size return-by-reference the total size of the new block + * @param total_weight return-by-reference the total weight of the new block * @param fee return-by-reference the total of fees from the included transactions * @param expected_reward return-by-reference the total reward awarded to the miner finding this block, including transaction fees * @param version hard fork version to use for consensus rules * * @return true */ - bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version); + bool fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version); /** * @brief get a list of all transactions in the pool @@ -249,7 +249,7 @@ namespace cryptonote void get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes = true) const; /** - * @brief get (size, fee, receive time) for all transaction in the pool + * @brief get (weight, fee, receive time) for all transaction in the pool * * @param txs return-by-reference that data * @param include_unrelayed_txes include unrelayed txes in the result @@ -362,22 +362,29 @@ namespace cryptonote */ size_t validate(uint8_t version); + /** + * @brief return the cookie + * + * @return the cookie + */ + uint64_t cookie() const { return m_cookie; } + /** - * @brief get the cumulative txpool size in bytes + * @brief get the cumulative txpool weight in bytes * - * @return the cumulative txpool size in bytes + * @return the cumulative txpool weight in bytes */ - size_t get_txpool_size() const; + size_t get_txpool_weight() const; /** - * @brief set the max cumulative txpool size in bytes + * @brief set the max cumulative txpool weight in bytes * - * @param bytes the max cumulative txpool size in bytes + * @param bytes the max cumulative txpool weight in bytes */ - void set_txpool_max_size(size_t bytes); + void set_txpool_max_weight(size_t bytes); #define CURRENT_MEMPOOL_ARCHIVE_VER 11 -#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 12 +#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 13 /** * @brief information about a single transaction @@ -386,6 +393,7 @@ namespace cryptonote { transaction tx; //!< the transaction size_t blob_size; //!< the transaction's size + size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount crypto::hash max_used_block_id; //!< the hash of the highest block referenced by an input uint64_t max_used_block_height; //!< the height of the highest block referenced by an input @@ -515,7 +523,7 @@ namespace cryptonote /** * @brief prune lowest fee/byte txes till we're not above bytes * - * if bytes is 0, use m_txpool_max_size + * if bytes is 0, use m_txpool_max_weight */ void prune(size_t bytes = 0); @@ -549,6 +557,8 @@ private: //!< container for transactions organized by fee per size and receive time sorted_tx_container m_txs_by_fee_and_receive_time; + std::atomic<uint64_t> m_cookie; //!< incremented at each change + /** * @brief get an iterator to a transaction in the sorted container * @@ -569,8 +579,8 @@ private: Blockchain& m_blockchain; //!< reference to the Blockchain object - size_t m_txpool_max_size; - size_t m_txpool_size; + size_t m_txpool_max_weight; + size_t m_txpool_weight; mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache; }; @@ -599,6 +609,9 @@ namespace boost if (version < 12) return; ar & td.do_not_relay; + if (version < 13) + return; + ar & td.weight; } } } diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index c39d67ceb..05f4189fb 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -57,7 +57,11 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) + { + for (const crypto::hash &h: hashes) + requested_hashes.insert(h); set_span_hashes(height, connection_id, hashes); + } } void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time) @@ -76,11 +80,19 @@ void block_queue::flush_spans(const boost::uuids::uuid &connection_id, bool all) block_map::iterator j = i++; if (j->connection_id == connection_id && (all || j->blocks.size() == 0)) { - blocks.erase(j); + erase_block(j); } } } +void block_queue::erase_block(block_map::iterator j) +{ + CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); + for (const crypto::hash &h: j->hashes) + requested_hashes.erase(h); + blocks.erase(j); +} + void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -92,7 +104,7 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con block_map::iterator j = i++; if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) { - blocks.erase(j); + erase_block(j); } } } @@ -106,7 +118,7 @@ bool block_queue::remove_span(uint64_t start_block_height, std::vector<crypto::h { if (hashes) *hashes = std::move(i->hashes); - blocks.erase(i); + erase_block(i); return true; } } @@ -121,7 +133,7 @@ void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t block_map::iterator j = i++; if (j->connection_id == connection_id && j->start_block_height <= start_block_height) { - blocks.erase(j); + erase_block(j); } } } @@ -160,16 +172,15 @@ std::string block_queue::get_overview() const return s; } +inline bool block_queue::requested_internal(const crypto::hash &hash) const +{ + return requested_hashes.find(hash) != requested_hashes.end(); +} + bool block_queue::requested(const crypto::hash &hash) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); - for (const auto &span: blocks) - { - for (const auto &h: span.hashes) - if (h == hash) - return true; - } - return false; + return requested_internal(hash); } std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) @@ -184,7 +195,7 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei uint64_t span_start_height = last_block_height - block_hashes.size() + 1; std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); - while (i != block_hashes.end() && requested(*i)) + while (i != block_hashes.end() && requested_internal(*i)) { ++i; ++span_start_height; @@ -256,8 +267,10 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui if (i->start_block_height == start_height && i->connection_id == connection_id) { span s = *i; - blocks.erase(i); + erase_block(i); s.hashes = std::move(hashes); + for (const crypto::hash &h: s.hashes) + requested_hashes.insert(h); blocks.insert(s); return; } diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 9059e89ac..9cce95075 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -33,6 +33,7 @@ #include <string> #include <vector> #include <set> +#include <unordered_set> #include <boost/thread/recursive_mutex.hpp> #include <boost/uuid/uuid.hpp> @@ -93,7 +94,12 @@ namespace cryptonote bool requested(const crypto::hash &hash) const; private: + void erase_block(block_map::iterator j); + inline bool requested_internal(const crypto::hash &hash) const; + + private: block_map blocks; mutable boost::recursive_mutex mutex; + std::unordered_set<crypto::hash> requested_hashes; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 56aa1dc06..c2c660e8c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1168,8 +1168,20 @@ skip: + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued"; if (ELPP->vRegistry()->allowed(el::Level::Debug, "sync-info")) timing_message += std::string(": ") + m_block_queue.get_overview(); - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() + if(m_core.get_target_blockchain_height() == 0){ + MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() << timing_message); + } else { + const int completition_percent = (m_core.get_current_blockchain_height() * 100 / m_core.get_target_blockchain_height()); + if(completition_percent < 99) {//printing completion percent only if % is < of 99 cause for 99 >= this is useless + MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() + << " (" << completition_percent << "% " << (m_core.get_target_blockchain_height() - m_core.get_current_blockchain_height()) + << " blocks remaining)" << timing_message); + } else { + MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() + << timing_message); + } + } } } } @@ -1671,11 +1683,6 @@ skip: fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; - // pre-serialize them - std::string fullBlob, fluffyBlob; - epee::serialization::store_t_to_binary(arg, fullBlob); - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - // sort peers between fluffy ones and others std::list<boost::uuids::uuid> fullConnections, fluffyConnections; m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) @@ -1697,8 +1704,18 @@ skip: }); // send fluffy ones first, we want to encourage people to run that - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + if (!fluffyConnections.empty()) + { + std::string fluffyBlob; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); + } + if (!fullConnections.empty()) + { + std::string fullBlob; + epee::serialization::store_t_to_binary(arg, fullBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + } return true; } @@ -1753,3 +1770,4 @@ skip: m_core.stop(); } } // namespace + diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 84004c3c6..b1c4b711d 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -28,7 +28,9 @@ set(blocksdat "") if(PER_BLOCK_CHECKPOINT) - if(APPLE) + if(APPLE AND DEPENDS) + add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} --target=x86_64-apple-darwin11 -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) + elseif(APPLE AND NOT DEPENDS) add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) else() add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 82ece62a9..f483ba6c9 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -239,11 +239,14 @@ int main(int argc, char const * argv[]) return 1; } + const char *env_rpc_login = nullptr; + const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login); + const bool use_rpc_env = !has_rpc_arg && (env_rpc_login = getenv("RPC_LOGIN")) != nullptr && strlen(env_rpc_login) > 0; boost::optional<tools::login> login{}; - if (command_line::has_arg(vm, arg.rpc_login)) + if (has_rpc_arg || use_rpc_env) { login = tools::login::parse( - command_line::get_arg(vm, arg.rpc_login), false, [](bool verify) { + has_rpc_arg ? command_line::get_arg(vm, arg.rpc_login) : std::string(env_rpc_login), false, [](bool verify) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index e2b42c806..6b6c88907 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -76,6 +76,7 @@ namespace { << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl << "POW hash: " << header.pow_hash << std::endl << "block size: " << header.block_size << std::endl + << "block weight: " << header.block_weight << std::endl << "num txes: " << header.num_txes << std::endl << "reward: " << cryptonote::print_money(header.reward); } @@ -532,6 +533,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u req.start_height = start_block_index; req.end_height = end_block_index; + req.fill_pow_hash = false; std::string fail_message = "Unsuccessful"; @@ -557,8 +559,8 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u if (!first) std::cout << std::endl; std::cout - << "height: " << header.height << ", timestamp: " << header.timestamp << ", difficulty: " << header.difficulty - << ", size: " << header.block_size << ", transactions: " << header.num_txes << std::endl + << "height: " << header.height << ", timestamp: " << header.timestamp + << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; @@ -857,8 +859,9 @@ bool t_rpc_command_executor::print_transaction_pool_long() { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << tx_info.tx_json << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -940,8 +943,9 @@ bool t_rpc_command_executor::print_transaction_pool_short() { { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -996,7 +1000,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0; std::string backlog_message; - const uint64_t full_reward_zone = ires.block_size_limit / 2; + const uint64_t full_reward_zone = ires.block_weight_limit / 2; if (res.pool_stats.bytes_total <= full_reward_zone) { backlog_message = "no backlog"; @@ -1701,8 +1705,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) cryptonote::COMMAND_RPC_GET_INFO::response ires; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request bhreq; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response bhres; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request fereq; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request fereq; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response feres; epee::json_rpc::error error_resp; std::string fail_message = "Problem fetching info"; @@ -1726,7 +1730,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) tools::fail_msg_writer() << make_error(fail_message, ires.status); return true; } - if (!m_rpc_server->on_get_per_kb_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_base_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, feres.status); return true; @@ -1743,6 +1747,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) bhreq.start_height = ires.height - nblocks; bhreq.end_height = ires.height - 1; + bhreq.fill_pow_hash = false; if (m_is_rpc) { if (!m_rpc_client->json_rpc_request(bhreq, bhres, "getblockheadersrange", fail_message.c_str())) @@ -1762,8 +1767,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) double avgdiff = 0; double avgnumtxes = 0; double avgreward = 0; - std::vector<uint64_t> sizes; - sizes.reserve(nblocks); + std::vector<uint64_t> weights; + weights.reserve(nblocks); uint64_t earliest = std::numeric_limits<uint64_t>::max(), latest = 0; std::vector<unsigned> major_versions(256, 0), minor_versions(256, 0); for (const auto &bhr: bhres.headers) @@ -1771,7 +1776,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff += bhr.difficulty; avgnumtxes += bhr.num_txes; avgreward += bhr.reward; - sizes.push_back(bhr.block_size); + weights.push_back(bhr.block_weight); static_assert(sizeof(bhr.major_version) == 1, "major_version expected to be uint8_t"); static_assert(sizeof(bhr.minor_version) == 1, "major_version expected to be uint8_t"); major_versions[(unsigned)bhr.major_version]++; @@ -1782,9 +1787,9 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff /= nblocks; avgnumtxes /= nblocks; avgreward /= nblocks; - uint64_t median_block_size = epee::misc_utils::median(sizes); + uint64_t median_block_weight = epee::misc_utils::median(weights); tools::msg_writer() << "Last " << nblocks << ": avg. diff " << (uint64_t)avgdiff << ", " << (latest - earliest) / nblocks << " avg sec/block, avg num txes " << avgnumtxes - << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block size " << median_block_size; + << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block weight " << median_block_weight; unsigned int max_major = 256, max_minor = 256; while (max_major > 0 && !major_versions[--max_major]); diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 6c09b0f18..3e2552230 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -43,6 +43,15 @@ using namespace epee; using namespace cryptonote; +static std::string extra_nonce_to_string(const cryptonote::tx_extra_nonce &extra_nonce) +{ + if (extra_nonce.nonce.size() == 9 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID) + return "encrypted payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + if (extra_nonce.nonce.size() == 33 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_PAYMENT_ID) + return "plaintext payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + return epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce); +} + static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fields) { std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; @@ -51,7 +60,7 @@ static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fi std::cout << "field " << n << ": "; if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get<cryptonote::tx_extra_padding>(fields[n]).size << " bytes"; else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key; - else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << extra_nonce_to_string(boost::get<cryptonote::tx_extra_nonce>(fields[n])); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root; else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get<cryptonote::tx_extra_additional_pub_keys>(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_mysterious_minergate>(fields[n]).data); @@ -168,9 +177,14 @@ int main(int argc, char* argv[]) std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; } - else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { - std::cout << "Parsed transaction:" << std::endl; +/* + if (tx.pruned) + std::cout << "Parsed pruned transaction:" << std::endl; + else +*/ + std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; bool parsed = cryptonote::parse_tx_extra(tx.extra, fields); diff --git a/src/device/device.cpp b/src/device/device.cpp index 983f59b60..8a8b40061 100644 --- a/src/device/device.cpp +++ b/src/device/device.cpp @@ -39,32 +39,60 @@ namespace hw { /* ======================================================================= */ /* SETUP */ - /* ======================================================================= */ - device& get_device(const std::string device_descriptor) { - - struct s_devices { - std::map<std::string, std::unique_ptr<device>> registry; - s_devices() : registry() { - hw::core::register_all(registry); - #ifdef HAVE_PCSC - hw::ledger::register_all(registry); - #endif - }; - }; - - static const s_devices devices; + /* ======================================================================= */ + + static std::unique_ptr<device_registry> registry; + + device_registry::device_registry(){ + hw::core::register_all(registry); + #ifdef HAVE_PCSC + hw::ledger::register_all(registry); + #endif + } + + bool device_registry::register_device(const std::string & device_name, device * hw_device){ + auto search = registry.find(device_name); + if (search != registry.end()){ + return false; + } + + registry.insert(std::make_pair(device_name, std::unique_ptr<device>(hw_device))); + return true; + } + + device& device_registry::get_device(const std::string & device_descriptor){ + // Device descriptor can contain further specs after first : + auto delim = device_descriptor.find(':'); + auto device_descriptor_lookup = device_descriptor; + if (delim != std::string::npos) { + device_descriptor_lookup = device_descriptor.substr(0, delim); + } - auto device = devices.registry.find(device_descriptor); - if (device == devices.registry.end()) { - MERROR("device not found in registry: '" << device_descriptor << "'\n" << - "known devices:"); - - for( const auto& sm_pair : devices.registry ) { + auto device = registry.find(device_descriptor_lookup); + if (device == registry.end()) { + MERROR("Device not found in registry: '" << device_descriptor << "'. Known devices: "); + for( const auto& sm_pair : registry ) { MERROR(" - " << sm_pair.first); } - throw std::runtime_error("device not found: "+ device_descriptor); + throw std::runtime_error("device not found: " + device_descriptor); } return *device->second; } + device& get_device(const std::string & device_descriptor) { + if (!registry){ + registry.reset(new device_registry()); + } + + return registry->get_device(device_descriptor); + } + + bool register_device(const std::string & device_name, device * hw_device){ + if (!registry){ + registry.reset(new device_registry()); + } + + return registry->register_device(device_name, hw_device); + } + } diff --git a/src/device/device.hpp b/src/device/device.hpp index 9df0cb39d..d14b8848c 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -125,7 +125,7 @@ namespace hw { /* ======================================================================= */ virtual bool get_public_address(cryptonote::account_public_address &pubkey) = 0; virtual bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) = 0; - virtual bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) = 0; + virtual bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) = 0; /* ======================================================================= */ /* SUB ADDRESS */ @@ -202,6 +202,17 @@ namespace hw { ~reset_mode() { hwref.set_mode(hw::device::NONE);} }; - device& get_device(const std::string device_descriptor) ; + class device_registry { + private: + std::map<std::string, std::unique_ptr<device>> registry; + + public: + device_registry(); + bool register_device(const std::string & device_name, device * hw_device); + device& get_device(const std::string & device_descriptor); + }; + + device& get_device(const std::string & device_descriptor); + bool register_device(const std::string & device_name, device * hw_device); } diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 0071f7d4f..a4f40e041 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -100,14 +100,14 @@ namespace hw { /* WALLET & ADDRESS */ /* ======================================================================= */ - bool device_default::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) { + bool device_default::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) { const crypto::secret_key &view_key = keys.m_view_secret_key; const crypto::secret_key &spend_key = keys.m_spend_secret_key; - tools::scrubbed_arr<char, sizeof(view_key) + sizeof(spend_key) + 1> data; + epee::mlocked<tools::scrubbed_arr<char, sizeof(view_key) + sizeof(spend_key) + 1>> data; memcpy(data.data(), &view_key, sizeof(view_key)); memcpy(data.data() + sizeof(view_key), &spend_key, sizeof(spend_key)); data[sizeof(data) - 1] = CHACHA8_KEY_TAIL; - crypto::generate_chacha_key(data.data(), sizeof(data), key); + crypto::generate_chacha_key(data.data(), sizeof(data), key, kdf_rounds); return true; } bool device_default::get_public_address(cryptonote::account_public_address &pubkey) { diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 771fbba72..8d841d9de 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -73,7 +73,7 @@ namespace hw { /* ======================================================================= */ bool get_public_address(cryptonote::account_public_address &pubkey) override; bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; - bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) override; + bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) override; /* ======================================================================= */ /* SUB ADDRESS */ diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index f7bf58531..9b5ea0fd7 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -136,7 +136,8 @@ namespace hw { } bool operator==(const crypto::key_derivation &d0, const crypto::key_derivation &d1) { - return !memcmp(&d0, &d1, sizeof(d0)); + static_assert(sizeof(crypto::key_derivation) == 32, "key_derivation must be 32 bytes"); + return !crypto_verify_32((const unsigned char*)&d0, (const unsigned char*)&d1); } /* ===================================================================== */ @@ -531,20 +532,20 @@ namespace hw { return true; } - bool device_ledger::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) { + bool device_ledger::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) { AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE crypto::chacha_key key_x; cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); - this->controle_device->generate_chacha_key(keys_x, key_x); + this->controle_device->generate_chacha_key(keys_x, key_x, kdf_rounds); #endif send_simple(INS_GET_CHACHA8_PREKEY); char prekey[200]; memmove(prekey, &this->buffer_recv[0], 200); - crypto::generate_chacha_key_prehashed(&prekey[0], sizeof(prekey), key); + crypto::generate_chacha_key_prehashed(&prekey[0], sizeof(prekey), key, kdf_rounds); #ifdef DEBUG_HWDEVICE hw::ledger::check32("generate_chacha_key_prehashed", "key", (char*)key_x.data(), (char*)key.data()); @@ -1353,7 +1354,7 @@ namespace hw { this->exchange(); //pseudoOuts - if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeSimpleBulletproof)) { + if (type == rct::RCTTypeSimple) { for ( i = 0; i < inputs_size; i++) { offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index c30a38aca..e6c6e5b52 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -156,7 +156,7 @@ namespace hw { /* ======================================================================= */ bool get_public_address(cryptonote::account_public_address &pubkey) override; bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; - bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) override; + bool generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) override; /* ======================================================================= */ diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index 03e0a7946..f11f442bc 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -91,7 +91,7 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str for (size_t n = 0; n < total; ++n) { std::string name = basename + "-" + std::to_string(n + 1); - wallets[n].reset(new tools::wallet2(nettype)); + wallets[n].reset(new tools::wallet2(nettype, 1, false)); wallets[n]->init(""); wallets[n]->generate(name, pwd_container->password(), rct::rct2sk(rct::skGen()), false, false, create_address_file); } @@ -101,11 +101,13 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str std::vector<crypto::public_key> pk(total); for (size_t n = 0; n < total; ++n) { + wallets[n]->decrypt_keys(pwd_container->password()); if (!tools::wallet2::verify_multisig_info(wallets[n]->get_multisig_info(), sk[n], pk[n])) { tools::fail_msg_writer() << tr("Failed to verify multisig info"); return false; } + wallets[n]->encrypt_keys(pwd_container->password()); } // make the wallets multisig diff --git a/src/mnemonics/chinese_simplified.h b/src/mnemonics/chinese_simplified.h index 1ae8c89d6..0566b1079 100644 --- a/src/mnemonics/chinese_simplified.h +++ b/src/mnemonics/chinese_simplified.h @@ -72,7 +72,10 @@ namespace Language class Chinese_Simplified: public Base
{
public:
- Chinese_Simplified(): Base("简体中文 (中国)", "Chinese (simplified)", std::vector<std::string>({
+ Chinese_Simplified(): Base("简体中文 (中国)", "Chinese (simplified)", {}, 1)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"的",
"一",
"是",
@@ -1699,8 +1702,8 @@ namespace Language "秒",
"浙",
"貌"
- }), 1)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/dutch.h b/src/mnemonics/dutch.h index c9806f450..801caf986 100644 --- a/src/mnemonics/dutch.h +++ b/src/mnemonics/dutch.h @@ -49,7 +49,10 @@ namespace Language class Dutch: public Base
{
public:
- Dutch(): Base("Nederlands", "Dutch", std::vector<std::string>({
+ Dutch(): Base("Nederlands", "Dutch", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"aalglad",
"aalscholver",
"aambeeld",
@@ -1676,8 +1679,8 @@ namespace Language "zwiep",
"zwijmel",
"zworen"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 7dd09ecb9..3d6338856 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -43,8 +43,11 @@ #include <vector> #include <unordered_map> #include <boost/algorithm/string.hpp> +#include "wipeable_string.h" +#include "misc_language.h" #include "crypto/crypto.h" // for declaration of crypto::secret_key #include <fstream> +#include "common/int-util.h" #include "mnemonics/electrum-words.h" #include <stdexcept> #include <boost/filesystem.hpp> @@ -70,11 +73,19 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "mnemonic" +namespace crypto +{ + namespace ElectrumWords + { + std::vector<const Language::Base*> get_language_list(); + } +} + namespace { - uint32_t create_checksum_index(const std::vector<std::string> &word_list, + uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, uint32_t unique_prefix_length); - bool checksum_test(std::vector<std::string> seed, uint32_t unique_prefix_length); + bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length); /*! * \brief Finds the word list that contains the seed words and puts the indices @@ -85,7 +96,7 @@ namespace * \param language Language instance pointer to write to after it is found. * \return true if all the words were present in some language false if not. */ - bool find_seed_language(const std::vector<std::string> &seed, + bool find_seed_language(const std::vector<epee::wipeable_string> &seed, bool has_checksum, std::vector<uint32_t> &matched_indices, Language::Base **language) { // If there's a new language added, add an instance of it here. @@ -106,17 +117,19 @@ namespace }); Language::Base *fallback = NULL; + std::vector<epee::wipeable_string>::const_iterator it2; + matched_indices.reserve(seed.size()); + // Iterate through all the languages and find a match for (std::vector<Language::Base*>::iterator it1 = language_instances.begin(); it1 != language_instances.end(); it1++) { - const std::unordered_map<std::string, uint32_t> &word_map = (*it1)->get_word_map(); - const std::unordered_map<std::string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t> &word_map = (*it1)->get_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); // To iterate through seed words - std::vector<std::string>::const_iterator it2; bool full_match = true; - std::string trimmed_word; + epee::wipeable_string trimmed_word; // Iterate through all the words and see if they're all present for (it2 = seed.begin(); it2 != seed.end(); it2++) { @@ -159,6 +172,7 @@ namespace return true; } // Some didn't match. Clear the index array. + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); matched_indices.clear(); } @@ -173,6 +187,7 @@ namespace } MINFO("No match found"); + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); return false; } @@ -182,12 +197,12 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return Checksum index */ - uint32_t create_checksum_index(const std::vector<std::string> &word_list, + uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, uint32_t unique_prefix_length) { - std::string trimmed_words = ""; + epee::wipeable_string trimmed_words = ""; - for (std::vector<std::string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) + for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) { if (it->length() > unique_prefix_length) { @@ -209,22 +224,22 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return True if the test passed false if not. */ - bool checksum_test(std::vector<std::string> seed, uint32_t unique_prefix_length) + bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length) { if (seed.empty()) return false; // The last word is the checksum. - std::string last_word = seed.back(); + epee::wipeable_string last_word = seed.back(); seed.pop_back(); - std::string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; + epee::wipeable_string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; - std::string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : + epee::wipeable_string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : checksum; - std::string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : + epee::wipeable_string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; bool ret = trimmed_checksum == trimmed_last_word; - MINFO("Checksum is %s" << (ret ? "valid" : "invalid")); + MINFO("Checksum is " << (ret ? "valid" : "invalid")); return ret; } } @@ -252,13 +267,12 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name) { - std::vector<std::string> seed; + std::vector<epee::wipeable_string> seed; - boost::algorithm::trim(words); - boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); + words.split(seed); if (len % 4) { @@ -283,6 +297,7 @@ namespace crypto } std::vector<uint32_t> matched_indices; + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0]));}); Language::Base *language; if (!find_seed_language(seed, has_checksum, matched_indices, &language)) { @@ -305,33 +320,33 @@ namespace crypto for (unsigned int i=0; i < seed.size() / 3; i++) { - uint32_t val; - uint32_t w1, w2, w3; - w1 = matched_indices[i*3]; - w2 = matched_indices[i*3 + 1]; - w3 = matched_indices[i*3 + 2]; + uint32_t w[4]; + w[1] = matched_indices[i*3]; + w[2] = matched_indices[i*3 + 1]; + w[3] = matched_indices[i*3 + 2]; - val = w1 + word_list_length * (((word_list_length - w1) + w2) % word_list_length) + - word_list_length * word_list_length * (((word_list_length - w2) + w3) % word_list_length); + w[0]= w[1] + word_list_length * (((word_list_length - w[1]) + w[2]) % word_list_length) + + word_list_length * word_list_length * (((word_list_length - w[2]) + w[3]) % word_list_length); - if (!(val % word_list_length == w1)) + if (!(w[0]% word_list_length == w[1])) { + memwipe(w, sizeof(w)); MERROR("Invalid seed: mumble mumble"); return false; } - dst.append((const char*)&val, 4); // copy 4 bytes to position + dst.append((const char*)&w[0], 4); // copy 4 bytes to position + memwipe(w, sizeof(w)); } if (len > 0 && duplicate) { const size_t expected = len * 3 / 32; - std::string wlist_copy = words; if (seed.size() == expected/2) { - dst.append(dst); // if electrum 12-word seed, duplicate - wlist_copy += ' '; - wlist_copy += words; + dst += ' '; // if electrum 12-word seed, duplicate + dst += dst; // if electrum 12-word seed, duplicate + dst.pop_back(); // trailing space } } @@ -345,10 +360,10 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name) { - std::string s; + epee::wipeable_string s; if (!words_to_bytes(words, s, sizeof(dst), true, language_name)) { MERROR("Invalid seed: failed to convert words to bytes"); @@ -370,100 +385,57 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name) { if (len % 4 != 0 || len == 0) return false; - Language::Base *language; - if (language_name == "English") - { - language = Language::Singleton<Language::English>::instance(); - } - else if (language_name == "Nederlands") - { - language = Language::Singleton<Language::Dutch>::instance(); - } - else if (language_name == "Français") - { - language = Language::Singleton<Language::French>::instance(); - } - else if (language_name == "Español") - { - language = Language::Singleton<Language::Spanish>::instance(); - } - else if (language_name == "Português") + const Language::Base *language = NULL; + const std::vector<const Language::Base*> language_list = crypto::ElectrumWords::get_language_list(); + for (const Language::Base *l: language_list) { - language = Language::Singleton<Language::Portuguese>::instance(); + if (language_name == l->get_language_name() || language_name == l->get_english_language_name()) + language = l; } - else if (language_name == "日本語") - { - language = Language::Singleton<Language::Japanese>::instance(); - } - else if (language_name == "Italiano") - { - language = Language::Singleton<Language::Italian>::instance(); - } - else if (language_name == "Deutsch") - { - language = Language::Singleton<Language::German>::instance(); - } - else if (language_name == "русский язык") - { - language = Language::Singleton<Language::Russian>::instance(); - } - else if (language_name == "简体中文 (中国)") - { - language = Language::Singleton<Language::Chinese_Simplified>::instance(); - } - else if (language_name == "Esperanto") - { - language = Language::Singleton<Language::Esperanto>::instance(); - } - else if (language_name == "Lojban") - { - language = Language::Singleton<Language::Lojban>::instance(); - } - else + if (!language) { return false; } const std::vector<std::string> &word_list = language->get_word_list(); // To store the words for random access to add the checksum word later. - std::vector<std::string> words_store; + std::vector<epee::wipeable_string> words_store; uint32_t word_list_length = word_list.size(); // 4 bytes -> 3 words. 8 digits base 16 -> 3 digits base 1626 - for (unsigned int i=0; i < len/4; i++, words += ' ') + for (unsigned int i=0; i < len/4; i++, words.push_back(' ')) { - uint32_t w1, w2, w3; - - uint32_t val; + uint32_t w[4]; - memcpy(&val, src + (i * 4), 4); + w[0] = SWAP32LE(*(const uint32_t*)(src + (i * 4))); - w1 = val % word_list_length; - w2 = ((val / word_list_length) + w1) % word_list_length; - w3 = (((val / word_list_length) / word_list_length) + w2) % word_list_length; + w[1] = w[0] % word_list_length; + w[2] = ((w[0] / word_list_length) + w[1]) % word_list_length; + w[3] = (((w[0] / word_list_length) / word_list_length) + w[2]) % word_list_length; - words += word_list[w1]; + words += word_list[w[1]]; words += ' '; - words += word_list[w2]; + words += word_list[w[2]]; words += ' '; - words += word_list[w3]; + words += word_list[w[3]]; + + words_store.push_back(word_list[w[1]]); + words_store.push_back(word_list[w[2]]); + words_store.push_back(word_list[w[3]]); - words_store.push_back(word_list[w1]); - words_store.push_back(word_list[w2]); - words_store.push_back(word_list[w3]); + memwipe(w, sizeof(w)); } - words.pop_back(); - words += (' ' + words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]); + words += words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]; return true; } - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name) { return bytes_to_words(src.data, sizeof(src), words, language_name); @@ -507,11 +479,10 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed) + bool get_is_old_style_seed(const epee::wipeable_string &seed) { - std::vector<std::string> word_list; - boost::algorithm::trim(seed); - boost::split(word_list, seed, boost::is_any_of(" "), boost::token_compress_on); + std::vector<epee::wipeable_string> word_list; + seed.split(word_list); return word_list.size() != (seed_length + 1); } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 856edb92a..5401b9779 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -44,6 +44,8 @@ #include <map> #include "crypto/crypto.h" // for declaration of crypto::secret_key +namespace epee { class wipeable_string; } + /*! * \namespace crypto * @@ -70,7 +72,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name); /*! * \brief Converts seed words to bytes (secret key). @@ -79,7 +81,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name); /*! @@ -90,7 +92,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name); /*! @@ -100,7 +102,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name); /*! @@ -115,7 +117,7 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed); + bool get_is_old_style_seed(const epee::wipeable_string &seed); /*! * \brief Returns the name of a language in English diff --git a/src/mnemonics/english.h b/src/mnemonics/english.h index ee087674d..d5c5594ef 100644 --- a/src/mnemonics/english.h +++ b/src/mnemonics/english.h @@ -49,7 +49,10 @@ namespace Language class English: public Base
{
public:
- English(): Base("English", "English", std::vector<std::string>({
+ English(): Base("English", "English", {}, 3)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abbey",
"abducts",
"ability",
@@ -1676,8 +1679,8 @@ namespace Language "zombie",
"zones",
"zoom"
- }), 3)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/english_old.h b/src/mnemonics/english_old.h index b31491646..e35b907df 100644 --- a/src/mnemonics/english_old.h +++ b/src/mnemonics/english_old.h @@ -51,7 +51,10 @@ namespace Language class EnglishOld: public Base
{
public:
- EnglishOld(): Base("EnglishOld", "English (old)", std::vector<std::string>({
+ EnglishOld(): Base("EnglishOld", "English (old)", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"like",
"just",
"love",
@@ -1678,8 +1681,8 @@ namespace Language "unseen",
"weapon",
"weary"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps(ALLOW_DUPLICATE_PREFIXES | ALLOW_SHORT_WORDS);
}
};
diff --git a/src/mnemonics/esperanto.h b/src/mnemonics/esperanto.h index a1d1a3f30..b0be235ed 100644 --- a/src/mnemonics/esperanto.h +++ b/src/mnemonics/esperanto.h @@ -58,7 +58,10 @@ namespace Language class Esperanto: public Base
{
public:
- Esperanto(): Base("Esperanto", "Esperanto", std::vector<std::string>({
+ Esperanto(): Base("Esperanto", "Esperanto", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abako",
"abdiki",
"abelo",
@@ -1685,8 +1688,8 @@ namespace Language "zorgi",
"zukino",
"zumilo",
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/french.h b/src/mnemonics/french.h index 7eaf45650..48ec46f78 100644 --- a/src/mnemonics/french.h +++ b/src/mnemonics/french.h @@ -49,7 +49,10 @@ namespace Language class French: public Base { public: - French(): Base("Français", "French", std::vector<std::string>({ + French(): Base("Français", "French", {}, 4) + { + static constexpr const char * const words[NWORDS] = + { "abandon", "abattre", "aboi", @@ -1676,8 +1679,8 @@ namespace Language "zinc", "zone", "zoom" - }), 4) - { + }; + set_words(words); populate_maps(); } }; diff --git a/src/mnemonics/german.h b/src/mnemonics/german.h index 8eff43523..883a173a3 100644 --- a/src/mnemonics/german.h +++ b/src/mnemonics/german.h @@ -51,7 +51,10 @@ namespace Language class German: public Base
{
public:
- German(): Base("Deutsch", "German", std::vector<std::string>({
+ German(): Base("Deutsch", "German", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"Abakus",
"Abart",
"abbilden",
@@ -1678,8 +1681,8 @@ namespace Language "Zündung",
"Zweck",
"Zyklop"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/italian.h b/src/mnemonics/italian.h index d5ecb74f4..57cdfa25e 100644 --- a/src/mnemonics/italian.h +++ b/src/mnemonics/italian.h @@ -51,7 +51,10 @@ namespace Language class Italian: public Base
{
public:
- Italian(): Base("Italiano", "Italian", std::vector<std::string>({
+ Italian(): Base("Italiano", "Italian", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abbinare",
"abbonato",
"abisso",
@@ -1678,8 +1681,8 @@ namespace Language "zolfo",
"zombie",
"zucchero"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/japanese.h b/src/mnemonics/japanese.h index f3b3e4924..5baabedf2 100644 --- a/src/mnemonics/japanese.h +++ b/src/mnemonics/japanese.h @@ -71,7 +71,10 @@ namespace Language class Japanese: public Base
{
public:
- Japanese(): Base("日本語", "Japanese", std::vector<std::string>({
+ Japanese(): Base("日本語", "Japanese", {}, 3)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"あいこくしん",
"あいさつ",
"あいだ",
@@ -1698,8 +1701,8 @@ namespace Language "ひさん",
"びじゅつかん",
"ひしょ"
- }), 3)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 2b0c37c6b..52e784cef 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -53,15 +53,20 @@ namespace Language * \param count How many characters to return.
* \return A string consisting of the first count characters in s.
*/
- inline std::string utf8prefix(const std::string &s, size_t count)
+ template<typename T>
+ inline T utf8prefix(const T &s, size_t count)
{
- std::string prefix = "";
- const char *ptr = s.c_str();
- while (count-- && *ptr)
+ T prefix = "";
+ size_t avail = s.size();
+ const char *ptr = s.data();
+ while (count-- && avail--)
{
prefix += *ptr++;
- while (((*ptr) & 0xc0) == 0x80)
+ while (avail && ((*ptr) & 0xc0) == 0x80)
+ {
prefix += *ptr++;
+ --avail;
+ }
}
return prefix;
}
@@ -78,9 +83,12 @@ namespace Language ALLOW_SHORT_WORDS = 1<<0,
ALLOW_DUPLICATE_PREFIXES = 1<<1,
};
- const std::vector<std::string> word_list; /*!< A pointer to the array of words */
- std::unordered_map<std::string, uint32_t> word_map; /*!< hash table to find word's index */
- std::unordered_map<std::string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
+ enum {
+ NWORDS = 1626
+ };
+ std::vector<std::string> word_list; /*!< A pointer to the array of words */
+ std::unordered_map<epee::wipeable_string, uint32_t> word_map; /*!< hash table to find word's index */
+ std::unordered_map<epee::wipeable_string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
std::string language_name; /*!< Name of language */
std::string english_language_name; /*!< Name of language */
uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
@@ -91,7 +99,7 @@ namespace Language {
int ii;
std::vector<std::string>::const_iterator it;
- if (word_list.size () != 1626)
+ if (word_list.size () != NWORDS)
throw std::runtime_error("Wrong word list length for " + language_name);
for (it = word_list.begin(), ii = 0; it != word_list.end(); it++, ii++)
{
@@ -103,7 +111,7 @@ namespace Language else
throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
}
- std::string trimmed;
+ epee::wipeable_string trimmed;
if (it->length() > unique_prefix_length)
{
trimmed = utf8prefix(*it, unique_prefix_length);
@@ -115,9 +123,9 @@ namespace Language if (trimmed_word_map.find(trimmed) != trimmed_word_map.end())
{
if (flags & ALLOW_DUPLICATE_PREFIXES)
- MWARNING("Duplicate prefix in " << language_name << " word list: " << trimmed);
+ MWARNING("Duplicate prefix in " << language_name << " word list: " << std::string(trimmed.data(), trimmed.size()));
else
- throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + trimmed);
+ throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + std::string(trimmed.data(), trimmed.size()));
}
trimmed_word_map[trimmed] = ii;
}
@@ -133,6 +141,12 @@ namespace Language virtual ~Base()
{
}
+ void set_words(const char * const words[])
+ {
+ word_list.resize(NWORDS);
+ for (size_t i = 0; i < NWORDS; ++i)
+ word_list[i] = words[i];
+ }
/*!
* \brief Returns a pointer to the word list.
* \return A pointer to the word list.
@@ -145,7 +159,7 @@ namespace Language * \brief Returns a pointer to the word map.
* \return A pointer to the word map.
*/
- const std::unordered_map<std::string, uint32_t>& get_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t>& get_word_map() const
{
return word_map;
}
@@ -153,7 +167,7 @@ namespace Language * \brief Returns a pointer to the trimmed word map.
* \return A pointer to the trimmed word map.
*/
- const std::unordered_map<std::string, uint32_t>& get_trimmed_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t>& get_trimmed_word_map() const
{
return trimmed_word_map;
}
diff --git a/src/mnemonics/lojban.h b/src/mnemonics/lojban.h index 0966a1169..5162a8ec9 100644 --- a/src/mnemonics/lojban.h +++ b/src/mnemonics/lojban.h @@ -56,7 +56,10 @@ namespace Language class Lojban: public Base
{
public:
- Lojban(): Base("Lojban", "Lojban", std::vector<std::string>({
+ Lojban(): Base("Lojban", "Lojban", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"backi",
"bacru",
"badna",
@@ -1683,8 +1686,8 @@ namespace Language "noltruti'u",
"samtci",
"snaxa'a",
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/portuguese.h b/src/mnemonics/portuguese.h index 0195389ce..af04f89c2 100644 --- a/src/mnemonics/portuguese.h +++ b/src/mnemonics/portuguese.h @@ -72,7 +72,10 @@ namespace Language class Portuguese: public Base
{
public:
- Portuguese(): Base("Português", "Portuguese", std::vector<std::string>({
+ Portuguese(): Base("Português", "Portuguese", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"abaular",
"abdominal",
"abeto",
@@ -1699,8 +1702,8 @@ namespace Language "zeloso",
"zenite",
"zumbi"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/russian.h b/src/mnemonics/russian.h index d5dd556ef..f3e70ede6 100644 --- a/src/mnemonics/russian.h +++ b/src/mnemonics/russian.h @@ -51,7 +51,10 @@ namespace Language class Russian: public Base
{
public:
- Russian(): Base("русский язык", "Russian", std::vector<std::string>({
+ Russian(): Base("русский язык", "Russian", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"абажур",
"абзац",
"абонент",
@@ -1678,8 +1681,8 @@ namespace Language "яхта",
"ячейка",
"ящик"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps();
}
};
diff --git a/src/mnemonics/spanish.h b/src/mnemonics/spanish.h index 51f38fede..4d7a896a6 100644 --- a/src/mnemonics/spanish.h +++ b/src/mnemonics/spanish.h @@ -72,7 +72,10 @@ namespace Language class Spanish: public Base
{
public:
- Spanish(): Base("Español", "Spanish", std::vector<std::string>({
+ Spanish(): Base("Español", "Spanish", {}, 4)
+ {
+ static constexpr const char * const words[NWORDS] =
+ {
"ábaco",
"abdomen",
"abeja",
@@ -1699,8 +1702,8 @@ namespace Language "risa",
"ritmo",
"rito"
- }), 4)
- {
+ };
+ set_words(words);
populate_maps(ALLOW_SHORT_WORDS);
}
};
diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index d85c47772..a0a788b7d 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -47,9 +47,12 @@ namespace cryptonote crypto::secret_key get_multisig_blinded_secret_key(const crypto::secret_key &key) { rct::keyV data; + data.reserve(2); data.push_back(rct::sk2rct(key)); data.push_back(multisig_salt); - return rct::rct2sk(rct::hash_to_scalar(data)); + crypto::secret_key result = rct::rct2sk(rct::hash_to_scalar(data)); + memwipe(&data[0], sizeof(rct::key)); + return result; } //----------------------------------------------------------------- void generate_multisig_N_N(const account_keys &keys, const std::vector<crypto::public_key> &spend_keys, std::vector<crypto::secret_key> &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey) diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 9390626a8..74924e4f4 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -937,7 +937,7 @@ namespace nodetool bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), epee::string_tools::num_to_string_fast(ipv4.port()), m_config.m_net_config.connection_timeout, - con); + con, m_bind_ip.empty() ? "0.0.0.0" : m_bind_ip); if(!res) { @@ -1002,7 +1002,7 @@ namespace nodetool bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), epee::string_tools::num_to_string_fast(ipv4.port()), m_config.m_net_config.connection_timeout, - con); + con, m_bind_ip.empty() ? "0.0.0.0" : m_bind_ip); if (!res) { bool is_priority = is_priority_node(na); @@ -1617,7 +1617,7 @@ namespace nodetool return false; } return true; - }); + }, m_bind_ip.empty() ? "0.0.0.0" : m_bind_ip); if(!r) { LOG_WARNING_CC(context, "Failed to call connect_async, network error."); diff --git a/src/ringct/CMakeLists.txt b/src/ringct/CMakeLists.txt index c8dcdca26..29f166a3b 100644 --- a/src/ringct/CMakeLists.txt +++ b/src/ringct/CMakeLists.txt @@ -30,11 +30,13 @@ set(ringct_basic_sources rctOps.cpp rctTypes.cpp rctCryptoOps.c + multiexp.cc bulletproofs.cc) set(ringct_basic_private_headers rctOps.h rctTypes.h + multiexp.h bulletproofs.h) monero_private_headers(ringct_basic diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index fd15ffbc4..abe4ef18d 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -30,14 +30,17 @@ #include <stdlib.h> #include <openssl/ssl.h> +#include <openssl/bn.h> #include <boost/thread/mutex.hpp> #include "misc_log_ex.h" #include "common/perf_timer.h" +#include "cryptonote_config.h" extern "C" { #include "crypto/crypto-ops.h" } #include "rctOps.h" +#include "multiexp.h" #include "bulletproofs.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -47,27 +50,99 @@ extern "C" #define PERF_TIMER_START_BP(x) PERF_TIMER_START_UNIT(x, 1000000) +#define STRAUS_SIZE_LIMIT 128 +#define PIPPENGER_SIZE_LIMIT 0 + namespace rct { static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b); -static rct::keyV vector_powers(rct::key x, size_t n); +static rct::keyV vector_powers(const rct::key &x, size_t n); +static rct::keyV vector_dup(const rct::key &x, size_t n); static rct::key inner_product(const rct::keyV &a, const rct::keyV &b); static constexpr size_t maxN = 64; -static rct::key Hi[maxN], Gi[maxN]; -static ge_dsmp Gprecomp[64], Hprecomp[64]; +static constexpr size_t maxM = BULLETPROOF_MAX_OUTPUTS; +static rct::key Hi[maxN*maxM], Gi[maxN*maxM]; +static ge_p3 Hi_p3[maxN*maxM], Gi_p3[maxN*maxM]; +static std::shared_ptr<straus_cached_data> straus_HiGi_cache; +static std::shared_ptr<pippenger_cached_data> pippenger_HiGi_cache; static const rct::key TWO = { {0x02, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; -static const rct::keyV oneN = vector_powers(rct::identity(), maxN); +static const rct::key MINUS_ONE = { { 0xec, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; +static const rct::key MINUS_INV_EIGHT = { { 0x74, 0xa4, 0x19, 0x7a, 0xf0, 0x7d, 0x0b, 0xf7, 0x05, 0xc2, 0xda, 0x25, 0x2b, 0x5c, 0x0b, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a } }; +static const rct::keyV oneN = vector_dup(rct::identity(), maxN); static const rct::keyV twoN = vector_powers(TWO, maxN); static const rct::key ip12 = inner_product(oneN, twoN); static boost::mutex init_mutex; +static inline rct::key multiexp(const std::vector<MultiexpData> &data, bool HiGi) +{ + if (HiGi) + { + static_assert(128 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); + return data.size() <= 128 ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, get_pippenger_c(data.size())); + } + else + return data.size() <= 64 ? straus(data, NULL, 0) : pippenger(data, NULL, get_pippenger_c(data.size())); +} + +static bool is_reduced(const rct::key &scalar) +{ + rct::key reduced = scalar; + sc_reduce32(reduced.bytes); + return scalar == reduced; +} + +static void addKeys_acc_p3(ge_p3 *acc_p3, const rct::key &a, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_scalarmult_p3(&p3, a.bytes, &p3); + ge_cached cached; + ge_p3_to_cached(&cached, acc_p3); + ge_p1p1 p1; + ge_add(&p1, &p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static void add_acc_p3(ge_p3 *acc_p3, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_cached cached; + ge_p3_to_cached(&cached, &p3); + ge_p1p1 p1; + ge_add(&p1, acc_p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static void sub_acc_p3(ge_p3 *acc_p3, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_cached cached; + ge_p3_to_cached(&cached, &p3); + ge_p1p1 p1; + ge_sub(&p1, acc_p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static rct::key scalarmultKey(const ge_p3 &P, const rct::key &a) +{ + ge_p2 R; + ge_scalarmult(&R, a.bytes, &P); + rct::key aP; + ge_tobytes(aP.bytes, &R); + return aP; +} + static rct::key get_exponent(const rct::key &base, size_t idx) { static const std::string salt("bulletproof"); std::string hashed = std::string((const char*)base.bytes, sizeof(base)) + salt + tools::get_varint_data(idx); - return rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + const rct::key e = rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + CHECK_AND_ASSERT_THROW_MES(!(e == rct::identity()), "Exponent is point at infinity"); + return e; } static void init_exponents() @@ -77,13 +152,27 @@ static void init_exponents() static bool init_done = false; if (init_done) return; - for (size_t i = 0; i < maxN; ++i) + std::vector<MultiexpData> data; + for (size_t i = 0; i < maxN*maxM; ++i) { Hi[i] = get_exponent(rct::H, i * 2); - rct::precomp(Hprecomp[i], Hi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Hi_p3[i], Hi[i].bytes) == 0, "ge_frombytes_vartime failed"); Gi[i] = get_exponent(rct::H, i * 2 + 1); - rct::precomp(Gprecomp[i], Gi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Gi_p3[i], Gi[i].bytes) == 0, "ge_frombytes_vartime failed"); + + data.push_back({rct::zero(), Gi[i]}); + data.push_back({rct::zero(), Hi[i]}); } + + straus_HiGi_cache = straus_init_cache(data, STRAUS_SIZE_LIMIT); + pippenger_HiGi_cache = pippenger_init_cache(data, PIPPENGER_SIZE_LIMIT); + + MINFO("Hi/Gi cache size: " << (sizeof(Hi)+sizeof(Gi))/1024 << " kB"); + MINFO("Hi_p3/Gi_p3 cache size: " << (sizeof(Hi_p3)+sizeof(Gi_p3))/1024 << " kB"); + MINFO("Straus cache size: " << straus_get_cache_size(straus_HiGi_cache)/1024 << " kB"); + MINFO("Pippenger cache size: " << pippenger_get_cache_size(pippenger_HiGi_cache)/1024 << " kB"); + size_t cache_size = (sizeof(Hi)+sizeof(Hi_p3))*2 + straus_get_cache_size(straus_HiGi_cache) + pippenger_get_cache_size(pippenger_HiGi_cache); + MINFO("Total cache size: " << cache_size/1024 << "kB"); init_done = true; } @@ -91,15 +180,16 @@ static void init_exponents() static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; - rct::addKeys3(term, a[i], Gprecomp[i], b[i], Hprecomp[i]); - rct::addKeys(res, res, term); + multiexp_data.emplace_back(a[i], Gi_p3[i]); + multiexp_data.emplace_back(b[i], Hi_p3[i]); } - return res; + return multiexp(multiexp_data, true); } /* Compute a custom vector-scalar commitment */ @@ -108,44 +198,24 @@ static rct::key vector_exponent_custom(const rct::keyV &A, const rct::keyV &B, c CHECK_AND_ASSERT_THROW_MES(A.size() == B.size(), "Incompatible sizes of A and B"); CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); CHECK_AND_ASSERT_THROW_MES(a.size() == A.size(), "Incompatible sizes of a and A"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; -#if 0 - // we happen to know where A and B might fall, so don't bother checking the rest - ge_dsmp *Acache = NULL, *Bcache = NULL; - ge_dsmp Acache_custom[1], Bcache_custom[1]; - if (Gi[i] == A[i]) - Acache = Gprecomp + i; - else if (i<32 && Gi[i+32] == A[i]) - Acache = Gprecomp + i + 32; - else - { - rct::precomp(Acache_custom[0], A[i]); - Acache = Acache_custom; - } - if (i == 0 && B[i] == Hi[0]) - Bcache = Hprecomp; - else - { - rct::precomp(Bcache_custom[0], B[i]); - Bcache = Bcache_custom; - } - rct::addKeys3(term, a[i], *Acache, b[i], *Bcache); -#else - ge_dsmp Acache, Bcache; - rct::precomp(Bcache, B[i]); - rct::addKeys3(term, a[i], A[i], b[i], Bcache); -#endif - rct::addKeys(res, res, term); + multiexp_data.resize(multiexp_data.size() + 1); + multiexp_data.back().scalar = a[i]; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, A[i].bytes) == 0, "ge_frombytes_vartime failed"); + multiexp_data.resize(multiexp_data.size() + 1); + multiexp_data.back().scalar = b[i]; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, B[i].bytes) == 0, "ge_frombytes_vartime failed"); } - return res; + return multiexp(multiexp_data, false); } /* Given a scalar, construct a vector of powers */ -static rct::keyV vector_powers(rct::key x, size_t n) +static rct::keyV vector_powers(const rct::key &x, size_t n) { rct::keyV res(n); if (n == 0) @@ -161,6 +231,24 @@ static rct::keyV vector_powers(rct::key x, size_t n) return res; } +/* Given a scalar, return the sum of its powers from 0 to n-1 */ +static rct::key vector_power_sum(const rct::key &x, size_t n) +{ + if (n == 0) + return rct::zero(); + rct::key res = rct::identity(); + if (n == 1) + return res; + rct::key prev = x; + for (size_t i = 1; i < n; ++i) + { + if (i > 1) + sc_mul(prev.bytes, prev.bytes, x.bytes); + sc_add(res.bytes, res.bytes, prev.bytes); + } + return res; +} + /* Given two scalar arrays, construct the inner product */ static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) { @@ -232,6 +320,12 @@ static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) return res; } +/* Create a vector from copies of a single value */ +static rct::keyV vector_dup(const rct::key &x, size_t N) +{ + return rct::keyV(N, x); +} + /* Exponentiate a curve vector by a scalar */ static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) { @@ -243,6 +337,17 @@ static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) return res; } +/* Get the sum of a vector's elements */ +static rct::key vector_sum(const rct::keyV &a) +{ + rct::key res = rct::zero(); + for (size_t i = 0; i < a.size(); ++i) + { + sc_add(res.bytes, res.bytes, a[i].bytes); + } + return res; +} + static rct::key switch_endianness(rct::key k) { std::reverse(k.bytes, k.bytes + sizeof(k)); @@ -345,6 +450,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) PERF_TIMER_START_BP(PROVE_v); rct::addKeys2(V, gamma, sv, rct::H); + V = rct::scalarmultKey(V, INV_EIGHT); PERF_TIMER_STOP(PROVE_v); PERF_TIMER_START_BP(PROVE_aLaR); @@ -380,12 +486,14 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); #endif +try_again: PERF_TIMER_START_BP(PROVE_step1); // PAPER LINES 38-39 rct::key alpha = rct::skGen(); rct::key ve = vector_exponent(aL, aR); rct::key A; rct::addKeys(A, ve, rct::scalarmultBase(alpha)); + A = rct::scalarmultKey(A, INV_EIGHT); // PAPER LINES 40-42 rct::keyV sL = rct::skvGen(N), sR = rct::skvGen(N); @@ -393,10 +501,23 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) ve = vector_exponent(sL, sR); rct::key S; rct::addKeys(S, ve, rct::scalarmultBase(rho)); + S = rct::scalarmultKey(S, INV_EIGHT); // PAPER LINES 43-45 rct::key y = hash_cache_mash(hash_cache, A, S); + if (y == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("y is 0, trying again"); + goto try_again; + } rct::key z = hash_cache = rct::hash_to_scalar(y); + if (z == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("z is 0, trying again"); + goto try_again; + } // Polynomial construction before PAPER LINE 46 rct::key t0 = rct::zero(); @@ -405,7 +526,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); + rct::key ip1y = vector_sum(yN); rct::key tmp; sc_muladd(t0.bytes, z.bytes, ip1y.bytes, t0.bytes); @@ -437,7 +558,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) PERF_TIMER_START_BP(PROVE_step2); const auto HyNsR = hadamard(yN, sR); - const auto vpIz = vector_scalar(oneN, z); + const auto vpIz = vector_dup(z, N); const auto vp2zsq = vector_scalar(twoN, zsq); const auto aL_vpIz = vector_subtract(aL, vpIz); const auto aR_vpIz = vector_add(aR, vpIz); @@ -454,11 +575,19 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 47-48 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - rct::key T1 = rct::addKeys(rct::scalarmultKey(rct::H, t1), rct::scalarmultBase(tau1)); - rct::key T2 = rct::addKeys(rct::scalarmultKey(rct::H, t2), rct::scalarmultBase(tau2)); + rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); + T1 = rct::scalarmultKey(T1, INV_EIGHT); + rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); + T2 = rct::scalarmultKey(T2, INV_EIGHT); // PAPER LINES 49-51 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } // PAPER LINES 52-53 rct::key taux = rct::zero(); @@ -500,7 +629,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) for (size_t i = 0; i < N; ++i) { Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi[i], yinvpow); + Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); aprime[i] = l[i]; bprime[i] = r[i]; @@ -525,13 +654,21 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 18-19 L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultKey(rct::H, tmp)); + rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); + L[round] = rct::scalarmultKey(L[round], INV_EIGHT); R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultKey(rct::H, tmp)); + rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); + R[round] = rct::scalarmultKey(R[round], INV_EIGHT); // PAPER LINES 21-22 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } // PAPER LINES 24-25 const rct::key winv = invert(w[round]); @@ -567,179 +704,540 @@ Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) return bulletproof_PROVE(sv, gamma); } -/* Given a range proof, determine if it is valid */ -bool bulletproof_VERIFY(const Bulletproof &proof) +/* Given a set of values v (0..2^N-1) and masks gamma, construct a range proof */ +Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) { + CHECK_AND_ASSERT_THROW_MES(sv.size() == gamma.size(), "Incompatible sizes of sv and gamma"); + CHECK_AND_ASSERT_THROW_MES(!sv.empty(), "sv is empty"); + for (const rct::key &sve: sv) + CHECK_AND_ASSERT_THROW_MES(is_reduced(sve), "Invalid sv input"); + for (const rct::key &g: gamma) + CHECK_AND_ASSERT_THROW_MES(is_reduced(g), "Invalid gamma input"); + init_exponents(); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "V does not have exactly one element"); - CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); - CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - CHECK_AND_ASSERT_MES(proof.L.size() == 6, false, "Proof is not for 64 bits"); + PERF_TIMER_UNIT(PROVE, 1000000); - const size_t logN = proof.L.size(); - const size_t N = 1 << logN; + constexpr size_t logN = 6; // log2(64) + constexpr size_t N = 1<<logN; + size_t M, logM; + for (logM = 0; (M = 1<<logM) <= maxM && M < sv.size(); ++logM); + CHECK_AND_ASSERT_THROW_MES(M <= maxM, "sv/gamma are too large"); + const size_t logMN = logM + logN; + const size_t MN = M * N; + + rct::keyV V(sv.size()); + rct::keyV aL(MN), aR(MN); + rct::key tmp; - // Reconstruct the challenges - PERF_TIMER_START_BP(VERIFY); - PERF_TIMER_START_BP(VERIFY_start); - rct::key hash_cache = rct::hash_to_scalar(proof.V[0]); - rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); + PERF_TIMER_START_BP(PROVE_v); + for (size_t i = 0; i < sv.size(); ++i) + { + rct::addKeys2(V[i], gamma[i], sv[i], rct::H); + V[i] = rct::scalarmultKey(V[i], INV_EIGHT); + } + PERF_TIMER_STOP(PROVE_v); + + PERF_TIMER_START_BP(PROVE_aLaR); + for (size_t j = 0; j < M; ++j) + { + for (size_t i = N; i-- > 0; ) + { + if (j >= sv.size()) + { + aL[j*N+i] = rct::zero(); + } + else if (sv[j][i/8] & (((uint64_t)1)<<(i%8))) + { + aL[j*N+i] = rct::identity(); + } + else + { + aL[j*N+i] = rct::zero(); + } + sc_sub(aR[j*N+i].bytes, aL[j*N+i].bytes, rct::identity().bytes); + } + } + PERF_TIMER_STOP(PROVE_aLaR); + + // DEBUG: Test to ensure this recovers the value +#ifdef DEBUG_BP + for (size_t j = 0; j < M; ++j) + { + uint64_t test_aL = 0, test_aR = 0; + for (size_t i = 0; i < N; ++i) + { + if (aL[j*N+i] == rct::identity()) + test_aL += ((uint64_t)1)<<i; + if (aR[j*N+i] == rct::zero()) + test_aR += ((uint64_t)1)<<i; + } + uint64_t v_test = 0; + if (j < sv.size()) + for (int n = 0; n < 8; ++n) v_test |= (((uint64_t)sv[j][n]) << (8*n)); + CHECK_AND_ASSERT_THROW_MES(test_aL == v_test, "test_aL failed"); + CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); + } +#endif + +try_again: + rct::key hash_cache = rct::hash_to_scalar(V); + + PERF_TIMER_START_BP(PROVE_step1); + // PAPER LINES 38-39 + rct::key alpha = rct::skGen(); + rct::key ve = vector_exponent(aL, aR); + rct::key A; + rct::addKeys(A, ve, rct::scalarmultBase(alpha)); + A = rct::scalarmultKey(A, INV_EIGHT); + + // PAPER LINES 40-42 + rct::keyV sL = rct::skvGen(MN), sR = rct::skvGen(MN); + rct::key rho = rct::skGen(); + ve = vector_exponent(sL, sR); + rct::key S; + rct::addKeys(S, ve, rct::scalarmultBase(rho)); + S = rct::scalarmultKey(S, INV_EIGHT); + + // PAPER LINES 43-45 + rct::key y = hash_cache_mash(hash_cache, A, S); + if (y == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("y is 0, trying again"); + goto try_again; + } rct::key z = hash_cache = rct::hash_to_scalar(y); - rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); - PERF_TIMER_STOP(VERIFY_start); + if (z == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("z is 0, trying again"); + goto try_again; + } - PERF_TIMER_START_BP(VERIFY_line_60); - // Reconstruct the challenges - rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); - PERF_TIMER_STOP(VERIFY_line_60); + // Polynomial construction by coefficients + const auto zMN = vector_dup(z, MN); + rct::keyV l0 = vector_subtract(aL, zMN); + const rct::keyV &l1 = sL; - PERF_TIMER_START_BP(VERIFY_line_61); - // PAPER LINE 61 - rct::key L61Left = rct::addKeys(rct::scalarmultBase(proof.taux), rct::scalarmultKey(rct::H, proof.t)); + // This computes the ugly sum/concatenation from PAPER LINE 65 + rct::keyV zero_twos(MN); + const rct::keyV zpow = vector_powers(z, M+2); + for (size_t i = 0; i < MN; ++i) + { + zero_twos[i] = rct::zero(); + for (size_t j = 1; j <= M; ++j) + { + if (i >= (j-1)*N && i < j*N) + { + CHECK_AND_ASSERT_THROW_MES(1+j < zpow.size(), "invalid zpow index"); + CHECK_AND_ASSERT_THROW_MES(i-(j-1)*N < twoN.size(), "invalid twoN index"); + sc_muladd(zero_twos[i].bytes, zpow[1+j].bytes, twoN[i-(j-1)*N].bytes, zero_twos[i].bytes); + } + } + } - rct::key k = rct::zero(); - const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - rct::key tmp, tmp2; - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - PERF_TIMER_STOP(VERIFY_line_61); + rct::keyV r0 = vector_add(aR, zMN); + const auto yMN = vector_powers(y, MN); + r0 = hadamard(r0, yMN); + r0 = vector_add(r0, zero_twos); + rct::keyV r1 = hadamard(yMN, sR); - PERF_TIMER_START_BP(VERIFY_line_61rl); - sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); - rct::key L61Right = rct::scalarmultKey(rct::H, tmp); + // Polynomial construction before PAPER LINE 46 + rct::key t1_1 = inner_product(l0, r1); + rct::key t1_2 = inner_product(l1, r0); + rct::key t1; + sc_add(t1.bytes, t1_1.bytes, t1_2.bytes); + rct::key t2 = inner_product(l1, r1); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "proof.V does not have exactly one element"); - tmp = rct::scalarmultKey(proof.V[0], zsq); - rct::addKeys(L61Right, L61Right, tmp); + PERF_TIMER_STOP(PROVE_step1); - tmp = rct::scalarmultKey(proof.T1, x); - rct::addKeys(L61Right, L61Right, tmp); + PERF_TIMER_START_BP(PROVE_step2); + // PAPER LINES 47-48 + rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); + + rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); + T1 = rct::scalarmultKey(T1, INV_EIGHT); + rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); + T2 = rct::scalarmultKey(T2, INV_EIGHT); + // PAPER LINES 49-51 + rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } + + // PAPER LINES 52-53 + rct::key taux; + sc_mul(taux.bytes, tau1.bytes, x.bytes); rct::key xsq; sc_mul(xsq.bytes, x.bytes, x.bytes); - tmp = rct::scalarmultKey(proof.T2, xsq); - rct::addKeys(L61Right, L61Right, tmp); - PERF_TIMER_STOP(VERIFY_line_61rl); - - if (!(L61Right == L61Left)) + sc_muladd(taux.bytes, tau2.bytes, xsq.bytes, taux.bytes); + for (size_t j = 1; j <= sv.size(); ++j) { - MERROR("Verification failure at step 1"); - return false; + CHECK_AND_ASSERT_THROW_MES(j+1 < zpow.size(), "invalid zpow index"); + sc_muladd(taux.bytes, zpow[j+1].bytes, gamma[j-1].bytes, taux.bytes); } + rct::key mu; + sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); - PERF_TIMER_START_BP(VERIFY_line_62); - // PAPER LINE 62 - rct::key P = rct::addKeys(proof.A, rct::scalarmultKey(proof.S, x)); - PERF_TIMER_STOP(VERIFY_line_62); + // PAPER LINES 54-57 + rct::keyV l = l0; + l = vector_add(l, vector_scalar(l1, x)); + rct::keyV r = r0; + r = vector_add(r, vector_scalar(r1, x)); + PERF_TIMER_STOP(PROVE_step2); - // Compute the number of rounds for the inner product - const size_t rounds = proof.L.size(); - CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + PERF_TIMER_START_BP(PROVE_step3); + rct::key t = inner_product(l, r); - PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 - // The inner product challenges are computed per round - rct::keyV w(rounds); - for (size_t i = 0; i < rounds; ++i) + // DEBUG: Test if the l and r vectors match the polynomial forms +#ifdef DEBUG_BP + rct::key test_t; + const rct::key t0 = inner_product(l0, r0); + sc_muladd(test_t.bytes, t1.bytes, x.bytes, t0.bytes); + sc_muladd(test_t.bytes, t2.bytes, xsq.bytes, test_t.bytes); + CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); +#endif + + // PAPER LINES 32-33 + rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); + if (x_ip == rct::zero()) { - w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + PERF_TIMER_STOP(PROVE_step3); + MINFO("x_ip is 0, trying again"); + goto try_again; } - PERF_TIMER_STOP(VERIFY_line_21_22); - PERF_TIMER_START_BP(VERIFY_line_24_25); - // Basically PAPER LINES 24-25 - // Compute the curvepoints from G[i] and H[i] - rct::key inner_prod = rct::identity(); + // These are used in the inner product rounds + size_t nprime = MN; + rct::keyV Gprime(MN); + rct::keyV Hprime(MN); + rct::keyV aprime(MN); + rct::keyV bprime(MN); + const rct::key yinv = invert(y); rct::key yinvpow = rct::identity(); - rct::key ypow = rct::identity(); + for (size_t i = 0; i < MN; ++i) + { + Gprime[i] = Gi[i]; + Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); + sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + aprime[i] = l[i]; + bprime[i] = r[i]; + } + rct::keyV L(logMN); + rct::keyV R(logMN); + int round = 0; + rct::keyV w(logMN); // this is the challenge x in the inner product protocol + PERF_TIMER_STOP(PROVE_step3); - PERF_TIMER_START_BP(VERIFY_line_24_25_invert); - const rct::key yinv = invert(y); - rct::keyV winv(rounds); - for (size_t i = 0; i < rounds; ++i) - winv[i] = invert(w[i]); - PERF_TIMER_STOP(VERIFY_line_24_25_invert); + PERF_TIMER_START_BP(PROVE_step4); + // PAPER LINE 13 + while (nprime > 1) + { + // PAPER LINE 15 + nprime /= 2; - for (size_t i = 0; i < N; ++i) + // PAPER LINES 16-17 + rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + + // PAPER LINES 18-19 + L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); + rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); + L[round] = rct::scalarmultKey(L[round], INV_EIGHT); + R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); + rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); + R[round] = rct::scalarmultKey(R[round], INV_EIGHT); + + // PAPER LINES 21-22 + w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } + + // PAPER LINES 24-25 + const rct::key winv = invert(w[round]); + Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); + Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); + + // PAPER LINES 28-29 + aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); + bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); + + ++round; + } + PERF_TIMER_STOP(PROVE_step4); + + // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) + return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); +} + +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma) +{ + CHECK_AND_ASSERT_THROW_MES(v.size() == gamma.size(), "Incompatible sizes of v and gamma"); + + // vG + gammaH + PERF_TIMER_START_BP(PROVE_v); + rct::keyV sv(v.size()); + for (size_t i = 0; i < v.size(); ++i) { - // Convert the index to binary IN REVERSE and construct the scalar exponent - rct::key g_scalar = proof.a; - rct::key h_scalar; - sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + sv[i] = rct::zero(); + sv[i].bytes[0] = v[i] & 255; + sv[i].bytes[1] = (v[i] >> 8) & 255; + sv[i].bytes[2] = (v[i] >> 16) & 255; + sv[i].bytes[3] = (v[i] >> 24) & 255; + sv[i].bytes[4] = (v[i] >> 32) & 255; + sv[i].bytes[5] = (v[i] >> 40) & 255; + sv[i].bytes[6] = (v[i] >> 48) & 255; + sv[i].bytes[7] = (v[i] >> 56) & 255; + } + PERF_TIMER_STOP(PROVE_v); + return bulletproof_PROVE(sv, gamma); +} + +/* Given a range proof, determine if it is valid */ +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) +{ + init_exponents(); + + PERF_TIMER_START_BP(VERIFY); + + // sanity and figure out which proof is longest + size_t max_length = 0; + for (const Bulletproof *p: proofs) + { + const Bulletproof &proof = *p; + + // check scalar range + CHECK_AND_ASSERT_MES(is_reduced(proof.taux), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.mu), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.a), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.b), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.t), false, "Input scalar not in range"); + + CHECK_AND_ASSERT_MES(proof.V.size() >= 1, false, "V does not have at least one element"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); + CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - for (size_t j = rounds; j-- > 0; ) + max_length = std::max(max_length, proof.L.size()); + } + CHECK_AND_ASSERT_MES(max_length < 32, false, "At least one proof is too large"); + size_t maxMN = 1u << max_length; + + const size_t logN = 6; + const size_t N = 1 << logN; + rct::key tmp; + + // setup weighted aggregates + rct::key Z0 = rct::identity(); + rct::key z1 = rct::zero(); + rct::key Z2 = rct::identity(); + rct::key z3 = rct::zero(); + rct::keyV z4(maxMN, rct::zero()), z5(maxMN, rct::zero()); + rct::key Y2 = rct::identity(), Y3 = rct::identity(), Y4 = rct::identity(); + rct::key y0 = rct::zero(), y1 = rct::zero(); + for (const Bulletproof *p: proofs) + { + const Bulletproof &proof = *p; + + size_t M, logM; + for (logM = 0; (M = 1<<logM) <= maxM && M < proof.V.size(); ++logM); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+logM, false, "Proof is not the expected size"); + const size_t MN = M*N; + rct::key weight = rct::skGen(); + + // Reconstruct the challenges + PERF_TIMER_START_BP(VERIFY_start); + rct::key hash_cache = rct::hash_to_scalar(proof.V); + rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); + CHECK_AND_ASSERT_MES(!(y == rct::zero()), false, "y == 0"); + rct::key z = hash_cache = rct::hash_to_scalar(y); + CHECK_AND_ASSERT_MES(!(z == rct::zero()), false, "z == 0"); + rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); + CHECK_AND_ASSERT_MES(!(x == rct::zero()), false, "x == 0"); + rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); + CHECK_AND_ASSERT_MES(!(x_ip == rct::zero()), false, "x_ip == 0"); + PERF_TIMER_STOP(VERIFY_start); + + PERF_TIMER_START_BP(VERIFY_line_61); + // PAPER LINE 61 + sc_muladd(y0.bytes, proof.taux.bytes, weight.bytes, y0.bytes); + + const rct::keyV zpow = vector_powers(z, M+3); + + rct::key k; + const rct::key ip1y = vector_power_sum(y, MN); + sc_mulsub(k.bytes, zpow[2].bytes, ip1y.bytes, rct::zero().bytes); + for (size_t j = 1; j <= M; ++j) + { + CHECK_AND_ASSERT_MES(j+2 < zpow.size(), false, "invalid zpow index"); + sc_mulsub(k.bytes, zpow[j+2].bytes, ip12.bytes, k.bytes); + } + PERF_TIMER_STOP(VERIFY_line_61); + + PERF_TIMER_START_BP(VERIFY_line_61rl_new); + sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(proof.V.size()); + sc_sub(tmp.bytes, proof.t.bytes, tmp.bytes); + sc_muladd(y1.bytes, tmp.bytes, weight.bytes, y1.bytes); + for (size_t j = 0; j < proof.V.size(); j++) { - size_t J = w.size() - j - 1; + sc_mul(tmp.bytes, zpow[j+2].bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.V[j]); + } + rct::addKeys(Y2, Y2, rct::scalarmultKey(multiexp(multiexp_data, false), weight)); + rct::key weight8; + sc_mul(weight8.bytes, weight.bytes, EIGHT.bytes); + sc_mul(tmp.bytes, x.bytes, weight8.bytes); + rct::addKeys(Y3, Y3, rct::scalarmultKey(proof.T1, tmp)); + rct::key xsq; + sc_mul(xsq.bytes, x.bytes, x.bytes); + sc_mul(tmp.bytes, xsq.bytes, weight8.bytes); + rct::addKeys(Y4, Y4, rct::scalarmultKey(proof.T2, tmp)); + PERF_TIMER_STOP(VERIFY_line_61rl_new); + + PERF_TIMER_START_BP(VERIFY_line_62); + // PAPER LINE 62 + sc_mul(tmp.bytes, x.bytes, EIGHT.bytes); + rct::addKeys(Z0, Z0, rct::scalarmultKey(rct::addKeys(rct::scalarmult8(proof.A), rct::scalarmultKey(proof.S, tmp)), weight)); + PERF_TIMER_STOP(VERIFY_line_62); + + // Compute the number of rounds for the inner product + const size_t rounds = logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_21_22); + // PAPER LINES 21-22 + // The inner product challenges are computed per round + rct::keyV w(rounds); + for (size_t i = 0; i < rounds; ++i) + { + w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + CHECK_AND_ASSERT_MES(!(w[i] == rct::zero()), false, "w[i] == 0"); + } + PERF_TIMER_STOP(VERIFY_line_21_22); + + PERF_TIMER_START_BP(VERIFY_line_24_25); + // Basically PAPER LINES 24-25 + // Compute the curvepoints from G[i] and H[i] + rct::key yinvpow = rct::identity(); + rct::key ypow = rct::identity(); + + PERF_TIMER_START_BP(VERIFY_line_24_25_invert); + const rct::key yinv = invert(y); + rct::keyV winv(rounds); + for (size_t i = 0; i < rounds; ++i) + winv[i] = invert(w[i]); + PERF_TIMER_STOP(VERIFY_line_24_25_invert); + + for (size_t i = 0; i < MN; ++i) + { + // Convert the index to binary IN REVERSE and construct the scalar exponent + rct::key g_scalar = proof.a; + rct::key h_scalar; + sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); - if ((i & (((size_t)1)<<j)) == 0) + for (size_t j = rounds; j-- > 0; ) { - sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); + size_t J = w.size() - j - 1; + + if ((i & (((size_t)1)<<j)) == 0) + { + sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); + } + else + { + sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); + } } - else + + // Adjust the scalars using the exponents from PAPER LINE 62 + sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); + CHECK_AND_ASSERT_MES(2+i/N < zpow.size(), false, "invalid zpow index"); + CHECK_AND_ASSERT_MES(i%N < twoN.size(), false, "invalid twoN index"); + sc_mul(tmp.bytes, zpow[2+i/N].bytes, twoN[i%N].bytes); + sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); + sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + + sc_muladd(z4[i].bytes, g_scalar.bytes, weight.bytes, z4[i].bytes); + sc_muladd(z5[i].bytes, h_scalar.bytes, weight.bytes, z5[i].bytes); + + if (i != MN-1) { - sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); + sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + sc_mul(ypow.bytes, ypow.bytes, y.bytes); } } - // Adjust the scalars using the exponents from PAPER LINE 62 - sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); - sc_mul(tmp.bytes, zsq.bytes, twoN[i].bytes); - sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); - sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + PERF_TIMER_STOP(VERIFY_line_24_25); - // Now compute the basepoint's scalar multiplication - // Each of these could be written as a multiexp operation instead - rct::addKeys3(tmp, g_scalar, Gprecomp[i], h_scalar, Hprecomp[i]); - rct::addKeys(inner_prod, inner_prod, tmp); + // PAPER LINE 26 + PERF_TIMER_START_BP(VERIFY_line_26_new); + multiexp_data.clear(); + multiexp_data.reserve(2*rounds); - if (i != N-1) + sc_muladd(z1.bytes, proof.mu.bytes, weight.bytes, z1.bytes); + for (size_t i = 0; i < rounds; ++i) { - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - sc_mul(ypow.bytes, ypow.bytes, y.bytes); + sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.L[i]); + sc_mul(tmp.bytes, winv[i].bytes, winv[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.R[i]); } + rct::key acc = multiexp(multiexp_data, false); + rct::addKeys(Z2, Z2, rct::scalarmultKey(acc, weight)); + sc_mulsub(tmp.bytes, proof.a.bytes, proof.b.bytes, proof.t.bytes); + sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); + sc_muladd(z3.bytes, tmp.bytes, weight.bytes, z3.bytes); + PERF_TIMER_STOP(VERIFY_line_26_new); } - PERF_TIMER_STOP(VERIFY_line_24_25); - - PERF_TIMER_START_BP(VERIFY_line_26); - // PAPER LINE 26 - rct::key pprime; - sc_sub(tmp.bytes, rct::zero().bytes, proof.mu.bytes); - rct::addKeys(pprime, P, rct::scalarmultBase(tmp)); - - for (size_t i = 0; i < rounds; ++i) - { - sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); - sc_mul(tmp2.bytes, winv[i].bytes, winv[i].bytes); -#if 1 - ge_dsmp cacheL, cacheR; - rct::precomp(cacheL, proof.L[i]); - rct::precomp(cacheR, proof.R[i]); - rct::addKeys3(tmp, tmp, cacheL, tmp2, cacheR); - rct::addKeys(pprime, pprime, tmp); -#else - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.L[i], tmp)); - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.R[i], tmp2)); -#endif - } - sc_mul(tmp.bytes, proof.t.bytes, x_ip.bytes); - rct::addKeys(pprime, pprime, rct::scalarmultKey(rct::H, tmp)); - PERF_TIMER_STOP(VERIFY_line_26); + // now check all proofs at once PERF_TIMER_START_BP(VERIFY_step2_check); - sc_mul(tmp.bytes, proof.a.bytes, proof.b.bytes); - sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); - tmp = rct::scalarmultKey(rct::H, tmp); - rct::addKeys(tmp, tmp, inner_prod); + ge_p3 check1; + ge_scalarmult_base(&check1, y0.bytes); + addKeys_acc_p3(&check1, y1, rct::H); + sub_acc_p3(&check1, Y2); + sub_acc_p3(&check1, Y3); + sub_acc_p3(&check1, Y4); + if (!ge_p3_is_point_at_infinity(&check1)) + { + MERROR("Verification failure at step 1"); + return false; + } + ge_p3 check2; + sc_sub(tmp.bytes, rct::zero().bytes, z1.bytes); + ge_double_scalarmult_base_vartime_p3(&check2, z3.bytes, &ge_p3_H, tmp.bytes); + add_acc_p3(&check2, Z0); + add_acc_p3(&check2, Z2); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(2 * maxMN); + for (size_t i = 0; i < maxMN; ++i) + { + sc_sub(tmp.bytes, rct::zero().bytes, z4[i].bytes); + multiexp_data.emplace_back(tmp, Gi_p3[i]); + sc_sub(tmp.bytes, rct::zero().bytes, z5[i].bytes); + multiexp_data.emplace_back(tmp, Hi_p3[i]); + } + add_acc_p3(&check2, multiexp(multiexp_data, true)); PERF_TIMER_STOP(VERIFY_step2_check); - if (!(pprime == tmp)) + + if (!ge_p3_is_point_at_infinity(&check2)) { MERROR("Verification failure at step 2"); return false; @@ -749,4 +1247,19 @@ bool bulletproof_VERIFY(const Bulletproof &proof) return true; } +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs) +{ + std::vector<const Bulletproof*> proof_pointers; + for (const Bulletproof &proof: proofs) + proof_pointers.push_back(&proof); + return bulletproof_VERIFY(proof_pointers); +} + +bool bulletproof_VERIFY(const Bulletproof &proof) +{ + std::vector<const Bulletproof*> proofs; + proofs.push_back(&proof); + return bulletproof_VERIFY(proofs); +} + } diff --git a/src/ringct/bulletproofs.h b/src/ringct/bulletproofs.h index 3061d272e..b86202ccc 100644 --- a/src/ringct/bulletproofs.h +++ b/src/ringct/bulletproofs.h @@ -40,7 +40,11 @@ namespace rct Bulletproof bulletproof_PROVE(const rct::key &v, const rct::key &gamma); Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma); +Bulletproof bulletproof_PROVE(const rct::keyV &v, const rct::keyV &gamma); +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma); bool bulletproof_VERIFY(const Bulletproof &proof); +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs); +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs); } diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc new file mode 100644 index 000000000..21957b94c --- /dev/null +++ b/src/ringct/multiexp.cc @@ -0,0 +1,665 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#include "misc_log_ex.h" +#include "common/perf_timer.h" +extern "C" +{ +#include "crypto/crypto-ops.h" +} +#include "common/aligned.h" +#include "rctOps.h" +#include "multiexp.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "multiexp" + +//#define MULTIEXP_PERF(x) x +#define MULTIEXP_PERF(x) + +#define RAW_MEMORY_BLOCK +//#define ALTERNATE_LAYOUT +//#define TRACK_STRAUS_ZERO_IDENTITY + +// per points us for N/B points (B point bands) +// raw alt 128/192 4096/192 4096/4096 +// 0 0 52.6 71 71.2 +// 0 1 53.2 72.2 72.4 +// 1 0 52.7 67 67.1 +// 1 1 52.8 70.4 70.2 + +// Pippenger: +// 1 2 3 4 5 6 7 8 9 bestN +// 2 555 598 621 804 1038 1733 2486 5020 8304 1 +// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 +// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 +// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 +// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 +// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 +// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 +// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 +// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 +// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 +// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 +// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 + +// 2 4 8 16 32 64 128 256 512 1024 2048 4096 +// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 +// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 +// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 +// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 + +// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip +// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip + +namespace rct +{ + +static inline bool operator<(const rct::key &k0, const rct::key&k1) +{ + for (int n = 31; n >= 0; --n) + { + if (k0.bytes[n] < k1.bytes[n]) + return true; + if (k0.bytes[n] > k1.bytes[n]) + return false; + } + return false; +} + +static inline rct::key div2(const rct::key &k) +{ + rct::key res; + int carry = 0; + for (int n = 31; n >= 0; --n) + { + int new_carry = (k.bytes[n] & 1) << 7; + res.bytes[n] = k.bytes[n] / 2 + carry; + carry = new_carry; + } + return res; +} + +static inline rct::key pow2(size_t n) +{ + CHECK_AND_ASSERT_THROW_MES(n < 256, "Invalid pow2 argument"); + rct::key res = rct::zero(); + res[n >> 3] |= 1<<(n&7); + return res; +} + +static inline int test(const rct::key &k, size_t n) +{ + if (n >= 256) return 0; + return k[n >> 3] & (1 << (n & 7)); +} + +static inline void add(ge_p3 &p3, const ge_cached &other) +{ + ge_p1p1 p1; + ge_add(&p1, &p3, &other); + ge_p1p1_to_p3(&p3, &p1); +} + +static inline void add(ge_p3 &p3, const ge_p3 &other) +{ + ge_cached cached; + ge_p3_to_cached(&cached, &other); + add(p3, cached); +} + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 1, "Not enough points"); + std::vector<size_t> heap(points); + for (size_t n = 0; n < points; ++n) + heap[n] = n; + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_cached cached; + ge_p3_to_cached(&cached, &data[index1].point); + ge_p1p1 p1; + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 0, "Not enough points"); + std::vector<size_t> heap; + heap.reserve(points); + for (size_t n = 0; n < points; ++n) + { + if (!(data[n].scalar == rct::zero()) && !ge_p3_is_point_at_infinity(&data[n].point)) + heap.push_back(n); + } + points = heap.size(); + if (points == 0) + return rct::identity(); + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + + if (points < 2) + { + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(div, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + ge_cached cached; + ge_p1p1 p1; + ge_p2 p2; + + MULTIEXP_PERF(PERF_TIMER_RESUME(div)); + while (1) + { + rct::key s1_2 = div2(data[index1].scalar); + if (!(data[index2].scalar < s1_2)) + break; + if (data[index1].scalar.bytes[0] & 1) + { + data.resize(data.size()+1); + data.back().scalar = rct::identity(); + data.back().point = data[index1].point; + heap.push_back(data.size() - 1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + data[index1].scalar = div2(data[index1].scalar); + ge_p3_to_p2(&p2, &data[index1].point); + ge_p2_dbl(&p1, &p2); + ge_p1p1_to_p3(&data[index1].point, &p1); + } + MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_p3_to_cached(&cached, &data[index1].point); + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +static constexpr unsigned int STRAUS_C = 4; + +struct straus_cached_data +{ +#ifdef RAW_MEMORY_BLOCK + size_t size; + ge_cached *multiples; + straus_cached_data(): size(0), multiples(NULL) {} + ~straus_cached_data() { aligned_free(multiples); } +#else + std::vector<std::vector<ge_cached>> multiples; +#endif +}; +#ifdef RAW_MEMORY_BLOCK +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)*((1<<STRAUS_C)-1)+((digit)-1)] +#else +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)+cache->size*((digit)-1)] +#endif +#else +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[j][digit-1] +#else +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[digit][j] +#endif +#endif + +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(multiples, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + std::shared_ptr<straus_cached_data> cache(new straus_cached_data()); + +#ifdef RAW_MEMORY_BLOCK + const size_t offset = cache->size; + cache->multiples = (ge_cached*)aligned_realloc(cache->multiples, sizeof(ge_cached) * ((1<<STRAUS_C)-1) * std::max(offset, N), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->multiples, "Out of memory"); + cache->size = N; + for (size_t j=offset;j<N;++j) + { + ge_p3_to_cached(&CACHE_OFFSET(cache, j, 1), &data[j].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &CACHE_OFFSET(cache, j, i-1)); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&CACHE_OFFSET(cache, j, i), &p3); + } + } +#else +#ifdef ALTERNATE_LAYOUT + const size_t offset = cache->multiples.size(); + cache->multiples.resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + { + cache->multiples[i].resize((1<<STRAUS_C)-1); + ge_p3_to_cached(&cache->multiples[i][0], &data[i].point); + for (size_t j=2;j<1<<STRAUS_C;++j) + { + ge_add(&p1, &data[i].point, &cache->multiples[i][j-2]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j-1], &p3); + } + } +#else + cache->multiples.resize(1<<STRAUS_C); + size_t offset = cache->multiples[1].size(); + cache->multiples[1].resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + ge_p3_to_cached(&cache->multiples[1][i], &data[i].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + cache->multiples[i].resize(std::max(offset, N)); + for (size_t j=offset;j<N;++j) + { + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &cache->multiples[i-1][j]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j], &p3); + } + } +#endif +#endif + MULTIEXP_PERF(PERF_TIMER_STOP(multiples)); + + return cache; +} + +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache) +{ + size_t sz = 0; +#ifdef RAW_MEMORY_BLOCK + sz += cache->size * sizeof(ge_cached) * ((1<<STRAUS_C)-1); +#else + for (const auto &e0: cache->multiples) + sz += e0.size() * sizeof(ge_cached); +#endif + return sz; +} + +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache, size_t STEP) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); + bool HiGi = cache != NULL; + STEP = STEP ? STEP : 192; + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + static constexpr unsigned int mask = (1<<STRAUS_C)-1; + std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + +#ifdef TRACK_STRAUS_ZERO_IDENTITY + MULTIEXP_PERF(PERF_TIMER_START_UNIT(skip, 1000000)); + std::vector<uint8_t> skip(data.size()); + for (size_t i = 0; i < data.size(); ++i) + skip[i] = data[i].scalar == rct::zero() || ge_p3_is_point_at_infinity(&data[i].point); + MULTIEXP_PERF(PERF_TIMER_STOP(skip)); +#endif + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(digits, 1000000)); + std::unique_ptr<uint8_t[]> digits{new uint8_t[256 * data.size()]}; + for (size_t j = 0; j < data.size(); ++j) + { + unsigned char bytes33[33]; + memcpy(bytes33, data[j].scalar.bytes, 32); + bytes33[32] = 0; + const unsigned char *bytes = bytes33; +#if 1 + static_assert(STRAUS_C == 4, "optimized version needs STRAUS_C == 4"); + unsigned int i; + for (i = 0; i < 256; i += 8, bytes++) + { + digits[j*256+i] = bytes[0] & 0xf; + digits[j*256+i+1] = (bytes[0] >> 1) & 0xf; + digits[j*256+i+2] = (bytes[0] >> 2) & 0xf; + digits[j*256+i+3] = (bytes[0] >> 3) & 0xf; + digits[j*256+i+4] = ((bytes[0] >> 4) | (bytes[1]<<4)) & 0xf; + digits[j*256+i+5] = ((bytes[0] >> 5) | (bytes[1]<<3)) & 0xf; + digits[j*256+i+6] = ((bytes[0] >> 6) | (bytes[1]<<2)) & 0xf; + digits[j*256+i+7] = ((bytes[0] >> 7) | (bytes[1]<<1)) & 0xf; + } +#elif 1 + for (size_t i = 0; i < 256; ++i) + digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; +#else + rct::key shifted = data[j].scalar; + for (size_t i = 0; i < 256; ++i) + { + digits[j*256+i] = shifted.bytes[0] & 0xf; + shifted = div2(shifted, (256-i)>>3); + } +#endif + } + MULTIEXP_PERF(PERF_TIMER_STOP(digits)); + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + size_t start_i = 0; + while (start_i < 256 && !(maxscalar < pow2(start_i))) + start_i += STRAUS_C; + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + ge_p3 res_p3 = ge_p3_identity; + + for (size_t start_offset = 0; start_offset < data.size(); start_offset += STEP) + { + const size_t num_points = std::min(data.size() - start_offset, STEP); + + ge_p3 band_p3 = ge_p3_identity; + size_t i = start_i; + if (!(i < STRAUS_C)) + goto skipfirst; + while (!(i < STRAUS_C)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &band_p3); + for (size_t j = 0; j < STRAUS_C; ++j) + { + ge_p2_dbl(&p1, &p2); + if (j == STRAUS_C - 1) + ge_p1p1_to_p3(&band_p3, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } +skipfirst: + i -= STRAUS_C; + for (size_t j = start_offset; j < start_offset + num_points; ++j) + { +#ifdef TRACK_STRAUS_ZERO_IDENTITY + if (skip[j]) + continue; +#endif + const uint8_t digit = digits[j*256+i]; + if (digit) + { + ge_add(&p1, &band_p3, &CACHE_OFFSET(local_cache, j, digit)); + ge_p1p1_to_p3(&band_p3, &p1); + } + } + } + + ge_p3_to_cached(&cached, &band_p3); + ge_add(&p1, &res_p3, &cached); + ge_p1p1_to_p3(&res_p3, &p1); + } + + rct::key res; + ge_p3_tobytes(res.bytes, &res_p3); + return res; +} + +size_t get_pippenger_c(size_t N) +{ +// uncached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 +// cached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 + if (N <= 2) return 1; + if (N <= 8) return 2; + if (N <= 16) return 3; + if (N <= 64) return 4; + if (N <= 128) return 5; + if (N <= 256) return 6; + if (N <= 1024) return 7; + if (N <= 2048) return 8; + return 9; +} + +struct pippenger_cached_data +{ + size_t size; + ge_cached *cached; + pippenger_cached_data(): size(0), cached(NULL) {} + ~pippenger_cached_data() { aligned_free(cached); } +}; + +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); + + cache->size = N; + cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); + for (size_t i = 0; i < N; ++i) + ge_p3_to_cached(&cache->cached[i], &data[i].point); + + MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); + return cache; +} + +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache) +{ + return cache->size * sizeof(*cache->cached); +} + +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t c) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + if (c == 0) + c = get_pippenger_c(data.size()); + CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); + + ge_p3 result = ge_p3_identity; + std::unique_ptr<ge_p3[]> buckets{new ge_p3[1<<c]}; + std::shared_ptr<pippenger_cached_data> local_cache = cache == NULL ? pippenger_init_cache(data) : cache; + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + { + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + } + size_t groups = 0; + while (groups < 256 && !(maxscalar < pow2(groups))) + ++groups; + groups = (groups + c - 1) / c; + + for (size_t k = groups; k-- > 0; ) + { + if (!ge_p3_is_point_at_infinity(&result)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &result); + for (size_t i = 0; i < c; ++i) + { + ge_p1p1 p1; + ge_p2_dbl(&p1, &p2); + if (i == c - 1) + ge_p1p1_to_p3(&result, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } + } + for (size_t i = 0; i < (1u<<c); ++i) + buckets[i] = ge_p3_identity; + + // partition scalars into buckets + for (size_t i = 0; i < data.size(); ++i) + { + unsigned int bucket = 0; + for (size_t j = 0; j < c; ++j) + if (test(data[i].scalar, k*c+j)) + bucket |= 1<<j; + if (bucket == 0) + continue; + CHECK_AND_ASSERT_THROW_MES(bucket < (1u<<c), "bucket overflow"); + if (!ge_p3_is_point_at_infinity(&buckets[bucket])) + { + add(buckets[bucket], local_cache->cached[i]); + } + else + buckets[bucket] = data[i].point; + } + + // sum the buckets + ge_p3 pail = ge_p3_identity; + for (size_t i = (1<<c)-1; i > 0; --i) + { + if (!ge_p3_is_point_at_infinity(&buckets[i])) + add(pail, buckets[i]); + if (!ge_p3_is_point_at_infinity(&pail)) + add(result, pail); + } + } + + rct::key res; + ge_p3_tobytes(res.bytes, &result); + return res; +} + +} diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h new file mode 100644 index 000000000..559ab664a --- /dev/null +++ b/src/ringct/multiexp.h @@ -0,0 +1,71 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#pragma once + +#ifndef MULTIEXP_H +#define MULTIEXP_H + +#include <vector> +#include "crypto/crypto.h" +#include "rctTypes.h" +#include "misc_log_ex.h" + +namespace rct +{ + +struct MultiexpData { + rct::key scalar; + ge_p3 point; + + MultiexpData() {} + MultiexpData(const rct::key &s, const ge_p3 &p): scalar(s), point(p) {} + MultiexpData(const rct::key &s, const rct::key &p): scalar(s) + { + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&point, p.bytes) == 0, "ge_frombytes_vartime failed"); + } +}; + +struct straus_cached_data; +struct pippenger_cached_data; + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data); +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data); +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache); +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache); +size_t get_pippenger_c(size_t N); +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t c = 0); + +} + +#endif diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 68cc43128..6c3c4500e 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -60,16 +60,26 @@ namespace rct { //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data) + { + if (ge_frombytes_vartime(P, data)) + return false; + ge_p2 R; + ge_scalarmult(&R, curveOrder().bytes, P); + key tmp; + ge_tobytes(tmp.bytes, &R); + return tmp == identity(); + } + //generates a random scalar which can be used as a secret key or mask void skGen(key &sk) { - sk = crypto::rand<key>(); - sc_reduce32(sk.bytes); + random32_unbiased(sk.bytes); } //generates a random scalar which can be used as a secret key or mask key skGen() { - key sk = crypto::rand<key>(); - sc_reduce32(sk.bytes); + key sk; + skGen(sk); return sk; } @@ -79,9 +89,8 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES(rows > 0, "0 keys requested"); keyV rv(rows); size_t i = 0; - crypto::rand(rows * sizeof(key), (uint8_t*)&rv[0]); for (i = 0 ; i < rows ; i++) { - sc_reduce32(rv[i].bytes); + skGen(rv[i]); } return rv; } @@ -195,15 +204,33 @@ namespace rct { //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a) { - ge_p3 A; ge_p2 R; - CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, H.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); - ge_scalarmult(&R, a.bytes, &A); + ge_scalarmult(&R, a.bytes, &ge_p3_H); key aP; ge_tobytes(aP.bytes, &R); return aP; } + //Computes 8P + key scalarmult8(const key & P) { + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); + ge_p2 p2; + ge_p3_to_p2(&p2, &p3); + ge_p1p1 p1; + ge_mul8(&p1, &p2); + ge_p1p1_to_p2(&p2, &p1); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + //Computes aL where L is the curve order + bool isInMainSubgroup(const key & a) { + ge_p3 p3; + return toPointCheckOrder(&p3, a.bytes); + } + //Curve addition / subtractions //for curve points: AB = A + B diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index f8889af5c..50645821c 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -63,6 +63,8 @@ namespace rct { static const key I = { {0x01, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key L = { {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; static const key G = { {0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } }; + static const key EIGHT = { {0x08, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; + static const key INV_EIGHT = { { 0x79, 0x2f, 0xdc, 0xe2, 0x29, 0xe5, 0x06, 0x61, 0xd0, 0xda, 0x1c, 0x7d, 0xb3, 0x9d, 0xd3, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06 } }; //Creates a zero scalar inline key zero() { return Z; } @@ -83,6 +85,7 @@ namespace rct { keyM keyMInit(size_t rows, size_t cols); //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data); //generates a random scalar which can be used as a secret key or mask key skGen(); @@ -119,6 +122,10 @@ namespace rct { key scalarmultKey(const key &P, const key &a); //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a); + // multiplies a point by 8 + key scalarmult8(const key & P); + // checks a is in the main subgroup (ie, not a small one) + bool isInMainSubgroup(const key & a); //Curve addition / subtractions diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index f74216ed4..fe0cd9c57 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -45,30 +45,6 @@ using namespace std; #define CHECK_AND_ASSERT_MES_L1(expr, ret, message) {if(!(expr)) {MCERROR("verify", message); return ret;}} namespace rct { - bool is_simple(int type) - { - switch (type) - { - case RCTTypeSimple: - case RCTTypeSimpleBulletproof: - return true; - default: - return false; - } - } - - bool is_bulletproof(int type) - { - switch (type) - { - case RCTTypeSimpleBulletproof: - case RCTTypeFullBulletproof: - return true; - default: - return false; - } - } - Bulletproof proveRangeBulletproof(key &C, key &mask, uint64_t amount) { mask = rct::skGen(); @@ -78,6 +54,15 @@ namespace rct { return proof; } + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) + { + masks = rct::skvGen(amounts.size()); + Bulletproof proof = bulletproof_PROVE(amounts, masks); + CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); + C = proof.V; + return proof; + } + bool verBulletproof(const Bulletproof &proof) { try { return bulletproof_VERIFY(proof); } @@ -85,6 +70,13 @@ namespace rct { catch (...) { return false; } } + bool verBulletproof(const std::vector<const Bulletproof*> &proofs) + { + try { return bulletproof_VERIFY(proofs); } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (...) { return false; } + } + //Borromean (c.f. gmax/andytoshi's paper) boroSig genBorromean(const key64 x, const key64 P1, const key64 P2, const bits indices) { key64 L[2], alpha; @@ -285,6 +277,7 @@ namespace rct { for (j = 0; j < dsRows; j++) { addKeys2(L, rv.ss[i][j], c_old, pk[i][j]); hashToPoint(Hi, pk[i][j]); + CHECK_AND_ASSERT_MES(!(Hi == rct::identity()), false, "Data hashed to point at infinity"); addKeys3(R, rv.ss[i][j], Hi, c_old, Ip[j].k); toHash[3 * j + 1] = pk[i][j]; toHash[3 * j + 2] = L; @@ -389,7 +382,7 @@ namespace rct { std::stringstream ss; binary_archive<true> ba(ss); CHECK_AND_ASSERT_THROW_MES(!rv.mixRing.empty(), "Empty mixRing"); - const size_t inputs = is_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); + const size_t inputs = is_rct_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); const size_t outputs = rv.ecdhInfo.size(); key prehash; CHECK_AND_ASSERT_THROW_MES(const_cast<rctSig&>(rv).serialize_rctsig_base(ba, inputs, outputs), @@ -398,7 +391,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeSimpleBulletproof || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeBulletproof) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -492,7 +485,9 @@ namespace rct { for (size_t j = 0; j < outPk.size(); j++) { sc_sub(sk[rows].bytes, sk[rows].bytes, outSk[j].mask.bytes); //subtract output masks in last row.. } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(sk.data(), sk.size() * sizeof(key)); + return result; } @@ -521,7 +516,9 @@ namespace rct { M[i][0] = pubs[i].dest; subKeys(M[i][1], pubs[i].mask, Cout); } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(&sk[0], sizeof(key)); + return result; } @@ -655,7 +652,7 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -665,13 +662,10 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES((kLRki && msout) || (!kLRki && !msout), "Only one of kLRki/msout is present"); rctSig rv; - rv.type = bulletproof ? RCTTypeFullBulletproof : RCTTypeFull; + rv.type = RCTTypeFull; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else - rv.p.rangeSigs.resize(destinations.size()); + rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i = 0; @@ -681,17 +675,10 @@ namespace rct { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, amounts[i]); - else - rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); + rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else - CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); + CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); @@ -721,12 +708,13 @@ namespace rct { ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev) { + const bool bulletproof = range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -742,35 +730,74 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeSimpleBulletproof : RCTTypeSimple; + rv.type = bulletproof ? RCTTypeBulletproof : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else + if (!bulletproof) rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i; keyV masks(destinations.size()); //sk mask.. outSk.resize(destinations.size()); - key sumout = zero(); for (i = 0; i < destinations.size(); i++) { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); - else + if (!bulletproof) rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else + if (!bulletproof) CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - + } + + rv.p.bulletproofs.clear(); + if (bulletproof) + { + std::vector<uint64_t> proof_amounts; + size_t n_amounts = outamounts.size(); + size_t amounts_proved = 0; + if (range_proof_type == RangeProofPaddedBulletproof) + { + rct::keyV C, masks; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + for (i = 0; i < outamounts.size(); ++i) + { + rv.outPk[i].mask = rct::scalarmult8(C[i]); + outSk[i].mask = masks[i]; + } + } + else while (amounts_proved < n_amounts) + { + size_t batch_size = 1; + if (range_proof_type == RangeProofMultiOutputBulletproof) + while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) + batch_size *= 2; + rct::keyV C, masks; + std::vector<uint64_t> batch_amounts(batch_size); + for (i = 0; i < batch_size; ++i) + batch_amounts[i] = outamounts[i + amounts_proved]; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + for (i = 0; i < batch_size; ++i) + { + rv.outPk[i + amounts_proved].mask = rct::scalarmult8(C[i]); + outSk[i + amounts_proved].mask = masks[i]; + } + amounts_proved += batch_size; + } + } + + key sumout = zero(); + for (i = 0; i < outSk.size(); ++i) + { sc_add(sumout.bytes, outSk[i].mask.bytes, sumout.bytes); //mask amount and mask @@ -818,7 +845,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, RangeProofBorromean, hwdev); } //RingCT protocol @@ -833,13 +860,10 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number bool verRct(const rctSig & rv, bool semantics) { PERF_TIMER(verRct); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "verRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "verRct called on non-full rctSig"); if (semantics) { - if (rv.type == RCTTypeFullBulletproof) - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); - else - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "full rctSig has not one MG"); } @@ -856,19 +880,13 @@ namespace rct { tools::threadpool::waiter waiter; std::deque<bool> results(rv.outPk.size(), false); DP("range proofs verified?"); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }, true); - } + for (size_t i = 0; i < rv.outPk.size(); i++) + tpool.submit(&waiter, [&, i] { results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); waiter.wait(&tpool); - for (size_t i = 0; i < rv.outPk.size(); ++i) { + for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); + LOG_PRINT_L1("Range proof verified failed for proof " << i); return false; } } @@ -902,17 +920,26 @@ namespace rct { //ver RingCT simple //assumes only post-rct style inputs (at least for max anonymity) - bool verRctSimple(const rctSig & rv, bool semantics) { + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rvv) { try { - PERF_TIMER(verRctSimple); + PERF_TIMER(verRctSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "verRctSimple called on non simple rctSig"); - if (semantics) + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + std::deque<bool> results; + std::vector<const Bulletproof*> proofs; + size_t max_non_bp_proofs = 0, offset = 0; + + for (const rctSig *rvp: rvv) { - if (rv.type == RCTTypeSimpleBulletproof) + CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); + const rctSig &rv = *rvp; + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + if (bulletproof) { - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == n_bulletproof_amounts(rv.p.bulletproofs), false, "Mismatched sizes of outPk and bulletproofs"); CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.p.MGs.size(), false, "Mismatched sizes of rv.p.pseudoOuts and rv.p.MGs"); CHECK_AND_ASSERT_MES(rv.pseudoOuts.empty(), false, "rv.pseudoOuts is not empty"); } @@ -923,28 +950,22 @@ namespace rct { CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.empty(), false, "rv.p.pseudoOuts is not empty"); } CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); - } - else - { - // semantics check is early, and mixRing/MGs aren't resolved yet - if (rv.type == RCTTypeSimpleBulletproof) - CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); - else - CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); - } - const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + if (!bulletproof) + max_non_bp_proofs += rv.p.rangeSigs.size(); + } - std::deque<bool> results(threads); - tools::threadpool& tpool = tools::threadpool::getInstance(); - tools::threadpool::waiter waiter; + results.resize(max_non_bp_proofs); + for (const rctSig *rvp: rvv) + { + const rctSig &rv = *rvp; - const keyV &pseudoOuts = is_bulletproof(rv.type) ? rv.p.pseudoOuts : rv.pseudoOuts; + const bool bulletproof = is_rct_bulletproof(rv.type); + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; - if (semantics) { key sumOutpks = identity(); for (size_t i = 0; i < rv.outPk.size(); i++) { - addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); + addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); } DP(sumOutpks); key txnFeeKey = scalarmultH(d2h(rv.txnFee)); @@ -952,52 +973,100 @@ namespace rct { key sumPseudoOuts = identity(); for (size_t i = 0 ; i < pseudoOuts.size() ; i++) { - addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); + addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); } DP(sumPseudoOuts); //check pseudoOuts vs Outs.. if (!equalKeys(sumPseudoOuts, sumOutpks)) { - LOG_PRINT_L1("Sum check failed"); - return false; + LOG_PRINT_L1("Sum check failed"); + return false; } - results.clear(); - results.resize(rv.outPk.size()); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }, true); + if (bulletproof) + { + for (size_t i = 0; i < rv.p.bulletproofs.size(); i++) + proofs.push_back(&rv.p.bulletproofs[i]); } - waiter.wait(&tpool); - - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); - return false; - } + else + { + for (size_t i = 0; i < rv.p.rangeSigs.size(); i++) + tpool.submit(&waiter, [&, i, offset] { results[i+offset] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); + offset += rv.p.rangeSigs.size(); } } - else { - const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); - - results.clear(); - results.resize(rv.mixRing.size()); - for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { - tpool.submit(&waiter, [&, i] { - results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); - }, true); + if (!proofs.empty() && !verBulletproof(proofs)) + { + LOG_PRINT_L1("Aggregate range proof verified failed"); + return false; + } + + waiter.wait(&tpool); + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("Range proof verified failed for proof " << i); + return false; } - waiter.wait(&tpool); + } - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("verRctMGSimple failed for input " << i); - return false; - } + return true; + } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (const std::exception &e) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple: " << e.what()); + return false; + } + catch (...) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple, but not an actual exception"); + return false; + } + } + + bool verRctSemanticsSimple(const rctSig & rv) + { + return verRctSemanticsSimple(std::vector<const rctSig*>(1, &rv)); + } + + //ver RingCT simple + //assumes only post-rct style inputs (at least for max anonymity) + bool verRctNonSemanticsSimple(const rctSig & rv) { + try + { + PERF_TIMER(verRctNonSemanticsSimple); + + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctNonSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + // semantics check is early, and mixRing/MGs aren't resolved yet + if (bulletproof) + CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); + else + CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); + + const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + + std::deque<bool> results(threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; + + const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); + + results.clear(); + results.resize(rv.mixRing.size()); + for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { + tpool.submit(&waiter, [&, i] { + results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); + }); + } + waiter.wait(&tpool); + + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("verRctMGSimple failed for input " << i); + return false; } } @@ -1006,12 +1075,12 @@ namespace rct { // we can get deep throws from ge_frombytes_vartime if input isn't valid catch (const std::exception &e) { - LOG_PRINT_L1("Error in verRct: " << e.what()); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple: " << e.what()); return false; } catch (...) { - LOG_PRINT_L1("Error in verRct, but not an actual exception"); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple, but not an actual exception"); return false; } } @@ -1027,7 +1096,7 @@ namespace rct { // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "decodeRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "decodeRct called on non-full rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1040,6 +1109,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1055,7 +1126,7 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1068,6 +1139,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1083,12 +1156,12 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeFullBulletproof || rv.type == RCTTypeSimpleBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); CHECK_AND_ASSERT_MES(k.size() == msout.c.size(), false, "Mismatched k/msout.c size"); - if (rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeFull) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "MGs not a single element"); } diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index 5a9b2dd44..ae8bb91d7 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -119,14 +119,16 @@ namespace rct { //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev); rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } - bool verRctSimple(const rctSig & rv, bool semantics); - static inline bool verRctSimple(const rctSig & rv) { return verRctSimple(rv, true) && verRctSimple(rv, false); } + bool verRctSemanticsSimple(const rctSig & rv); + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rv); + bool verRctNonSemanticsSimple(const rctSig & rv); + static inline bool verRctSimple(const rctSig & rv) { return verRctSemanticsSimple(rv) && verRctNonSemanticsSimple(rv); } xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 5650b3ba1..90ed65df0 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -28,6 +28,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "misc_log_ex.h" +#include "cryptonote_config.h" #include "rctTypes.h" using namespace crypto; using namespace std; @@ -209,4 +211,90 @@ namespace rct { return vali; } + bool is_rct_simple(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_bulletproof(int type) + { + switch (type) + { + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_borromean(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeFull: + return true; + default: + return false; + } + } + + size_t n_bulletproof_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.V.size() <= (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() * 2 > (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() > 0, 0, "Empty bulletproof"); + return proof.V.size(); + } + + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + + size_t n_bulletproof_max_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + return 1 << (proof.L.size() - 6); + } + + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_max_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 844291d0c..18290637b 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -36,6 +36,7 @@ #include <vector> #include <iostream> #include <cinttypes> +#include <sodium/crypto_verify_32.h> extern "C" { #include "crypto/crypto-ops.h" @@ -81,7 +82,7 @@ namespace rct { unsigned char operator[](int i) const { return bytes[i]; } - bool operator==(const key &k) const { return !memcmp(bytes, k.bytes, sizeof(bytes)); } + bool operator==(const key &k) const { return !crypto_verify_32(bytes, k.bytes); } unsigned char bytes[32]; }; typedef std::vector<key> keyV; //vector of keys @@ -189,6 +190,8 @@ namespace rct { Bulletproof() {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} + Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): + V(V), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} BEGIN_SERIALIZE_OBJECT() // Commitments aren't saved, they're restored via outPk @@ -210,6 +213,11 @@ namespace rct { END_SERIALIZE() }; + size_t n_bulletproof_amounts(const Bulletproof &proof); + size_t n_bulletproof_max_amounts(const Bulletproof &proof); + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs); + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs); + //A container to hold all signatures necessary for RingCT // rangeSigs holds all the rangeproof data of a transaction // MG holds the MLSAG signature of a transaction @@ -221,9 +229,9 @@ namespace rct { RCTTypeNull = 0, RCTTypeFull = 1, RCTTypeSimple = 2, - RCTTypeFullBulletproof = 3, - RCTTypeSimpleBulletproof = 4, + RCTTypeBulletproof = 3, }; + enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; struct rctSigBase { uint8_t type; key message; @@ -240,7 +248,7 @@ namespace rct { FIELD(type) if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -301,21 +309,25 @@ namespace rct { { if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; - if (type == RCTTypeSimpleBulletproof || type == RCTTypeFullBulletproof) + if (type == RCTTypeBulletproof) { + uint32_t nbp = bulletproofs.size(); + FIELD(nbp) ar.tag("bp"); ar.begin_array(); - PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, bulletproofs); - if (bulletproofs.size() != outputs) + if (nbp > outputs) return false; - for (size_t i = 0; i < outputs; ++i) + PREPARE_CUSTOM_VECTOR_SERIALIZATION(nbp, bulletproofs); + for (size_t i = 0; i < nbp; ++i) { FIELDS(bulletproofs[i]) - if (outputs - i > 1) + if (nbp - i > 1) ar.delimit_array(); } + if (n_bulletproof_max_amounts(bulletproofs) < outputs) + return false; ar.end_array(); } else @@ -338,7 +350,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -356,7 +368,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -382,7 +394,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeSimpleBulletproof) + if (type == RCTTypeBulletproof) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -403,6 +415,16 @@ namespace rct { }; struct rctSig: public rctSigBase { rctSigPrunable p; + + keyV& get_pseudo_outs() + { + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + } + + keyV const& get_pseudo_outs() const + { + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + } }; //other basepoint H = toPoint(cn_fast_hash(G)), G the basepoint @@ -506,24 +528,28 @@ namespace rct { //int[64] to uint long long xmr_amount b2d(bits amountb); - static inline const rct::key pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } - static inline const rct::key sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } - static inline const rct::key ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } - static inline const rct::key hash2rct(const crypto::hash &h) { return (const rct::key&)h; } - static inline const crypto::public_key rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } - static inline const crypto::secret_key rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } - static inline const crypto::key_image rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } - static inline const crypto::hash rct2hash(const rct::key &k) { return (const crypto::hash&)k; } - static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return memcmp(&k0, &k1, 32); } + bool is_rct_simple(int type); + bool is_rct_bulletproof(int type); + bool is_rct_borromean(int type); + + static inline const rct::key &pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } + static inline const rct::key &sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } + static inline const rct::key &ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } + static inline const rct::key &hash2rct(const crypto::hash &h) { return (const rct::key&)h; } + static inline const crypto::public_key &rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } + static inline const crypto::secret_key &rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } + static inline const crypto::key_image &rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } + static inline const crypto::hash &rct2hash(const rct::key &k) { return (const crypto::hash&)k; } + static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } + static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } } namespace cryptonote { - static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } - static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } + static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } } namespace rct { diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0a6daf8f0..a85b94541 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -195,10 +195,10 @@ namespace cryptonote res.stagenet = m_nettype == STAGENET; res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; - res.start_time = (uint64_t)m_core.get_start_time(); + res.start_time = m_restricted ? 0 : (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); res.offline = m_core.offline(); res.bootstrap_daemon_address = m_bootstrap_daemon_address; @@ -364,49 +364,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) - { - PERF_TIMER(on_get_random_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS>(invoke_http_mode::BIN, "/getrandom_outs.bin", req, res, r)) - return r; - - res.status = "Failed"; - - if (m_restricted) - { - if (req.amounts.size() > 100 || req.outs_count > MAX_RESTRICTED_FAKE_OUTS_COUNT) - { - res.status = "Too many outs requested"; - return true; - } - } - - if(!m_core.get_random_outs_for_amounts(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount outs_for_amount; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - std::for_each(res.outs.begin(), res.outs.end(), [&](outs_for_amount& ofa) - { - ss << "[" << ofa.amount << "]:"; - CHECK_AND_ASSERT_MES(ofa.outs.size(), ;, "internal error: ofa.outs.size() is empty for amount " << ofa.amount); - std::for_each(ofa.outs.begin(), ofa.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - }); - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) { PERF_TIMER(on_get_outs_bin); @@ -476,34 +433,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) - { - PERF_TIMER(on_get_random_rct_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS>(invoke_http_mode::BIN, "/getrandom_rctouts.bin", req, res, r)) - return r; - - res.status = "Failed"; - if(!m_core.get_random_rct_outs(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry out_entry; - CHECK_AND_ASSERT_MES(res.outs.size(), true, "internal error: res.outs.size() is empty"); - std::for_each(res.outs.begin(), res.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) { PERF_TIMER(on_get_indexes); @@ -850,7 +779,13 @@ namespace cryptonote boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - if(!m_core.get_miner().start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + cryptonote::miner &miner= m_core.get_miner(); + if (miner.is_mining()) + { + res.status = "Already mining"; + return true; + } + if(!miner.start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) { res.status = "Failed, mining not started"; LOG_PRINT_L0(res.status); @@ -1313,7 +1248,7 @@ namespace cryptonote response.hash = string_tools::pod_to_hex(hash); response.difficulty = m_core.get_blockchain_storage().block_difficulty(height); response.reward = get_block_reward(blk); - response.block_size = m_core.get_blockchain_storage().get_db().get_block_size(height); + response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; return true; @@ -1646,8 +1581,8 @@ namespace cryptonote res.stagenet = m_nettype == STAGENET; res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; res.start_time = (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); @@ -1839,14 +1774,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) { - PERF_TIMER(on_get_per_kb_fee_estimate); + PERF_TIMER(on_get_base_fee_estimate); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BASE_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) return r; - res.fee = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.grace_blocks); + res.fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.grace_blocks); + res.quantization_mask = Blockchain::get_fee_quantization_mask(); res.status = CORE_RPC_STATUS_OK; return true; } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 166020d01..3ba882b23 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -87,11 +87,7 @@ namespace cryptonote MAP_URI_AUTO_BIN2("/get_hashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES) - MAP_URI_AUTO_BIN2("/get_random_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) - MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) MAP_URI_AUTO_BIN2("/get_outs.bin", on_get_outs_bin, COMMAND_RPC_GET_OUTPUTS_BIN) - MAP_URI_AUTO_BIN2("/get_random_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) - MAP_URI_AUTO_BIN2("/getrandom_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) MAP_URI_AUTO_JON2("/get_transactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/get_alt_blocks_hashes", on_get_alt_blocks_hashes, COMMAND_RPC_GET_ALT_BLOCKS_HASHES) @@ -150,7 +146,7 @@ namespace cryptonote MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) MAP_JON_RPC_WE("get_version", on_get_version, COMMAND_RPC_GET_VERSION) MAP_JON_RPC_WE_IF("get_coinbase_tx_sum", on_get_coinbase_tx_sum, COMMAND_RPC_GET_COINBASE_TX_SUM, !m_restricted) - MAP_JON_RPC_WE("get_fee_estimate", on_get_per_kb_fee_estimate, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE) + MAP_JON_RPC_WE("get_fee_estimate", on_get_base_fee_estimate, COMMAND_RPC_GET_BASE_FEE_ESTIMATE) MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains, COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted) MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) @@ -171,10 +167,8 @@ namespace cryptonote bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); - bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res); bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res); - bool on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res); bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); @@ -214,7 +208,7 @@ namespace cryptonote bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp); bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp); bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp); - bool on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); + bool on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index e2d120cad..1a84ee614 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -48,8 +48,8 @@ namespace cryptonote // whether they can talk to a given daemon without having to know in // advance which version they will stop working with // Don't go over 32767 for any of these -#define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 21 +#define CORE_RPC_VERSION_MAJOR 2 +#define CORE_RPC_VERSION_MINOR 0 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -680,50 +680,6 @@ namespace cryptonote }; }; //----------------------------------------------- - struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS - { - struct request - { - std::vector<uint64_t> amounts; - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amounts) - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t global_amount_index; - crypto::public_key out_key; - }; -#pragma pack(pop) - - struct outs_for_amount - { - uint64_t amount; - std::list<out_entry> outs; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - END_KV_SERIALIZE_MAP() - }; - - struct response - { - std::vector<outs_for_amount> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; - //----------------------------------------------- struct get_outputs_out { uint64_t amount; @@ -818,39 +774,6 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; - - struct COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS - { - struct request - { - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t amount; - uint64_t global_amount_index; - crypto::public_key out_key; - rct::key commitment; - }; -#pragma pack(pop) - - struct response - { - std::list<out_entry> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; //----------------------------------------------- struct COMMAND_RPC_SEND_RAW_TX { @@ -959,7 +882,9 @@ namespace cryptonote std::string top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; uint64_t free_space; bool offline; @@ -990,7 +915,9 @@ namespace cryptonote KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE(block_size_limit) + KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0) KV_SERIALIZE(block_size_median) + KV_SERIALIZE_OPT(block_weight_median, (uint64_t)0) KV_SERIALIZE(start_time) KV_SERIALIZE(free_space) KV_SERIALIZE(offline) @@ -1195,6 +1122,7 @@ namespace cryptonote difficulty_type difficulty; uint64_t reward; uint64_t block_size; + uint64_t block_weight; uint64_t num_txes; std::string pow_hash; @@ -1211,6 +1139,7 @@ namespace cryptonote KV_SERIALIZE(difficulty) KV_SERIALIZE(reward) KV_SERIALIZE(block_size) + KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) END_KV_SERIALIZE_MAP() @@ -1451,6 +1380,7 @@ namespace cryptonote std::string id_hash; std::string tx_json; // TODO - expose this data directly uint64_t blob_size; + uint64_t weight; uint64_t fee; std::string max_used_block_id_hash; uint64_t max_used_block_height; @@ -1468,6 +1398,7 @@ namespace cryptonote KV_SERIALIZE(id_hash) KV_SERIALIZE(tx_json) KV_SERIALIZE(blob_size) + KV_SERIALIZE_OPT(weight, (uint64_t)0) KV_SERIALIZE(fee) KV_SERIALIZE(max_used_block_id_hash) KV_SERIALIZE(max_used_block_height) @@ -1564,7 +1495,7 @@ namespace cryptonote struct tx_backlog_entry { - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t time_in_pool; }; @@ -2102,7 +2033,7 @@ namespace cryptonote }; }; - struct COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE + struct COMMAND_RPC_GET_BASE_FEE_ESTIMATE { struct request { @@ -2117,11 +2048,13 @@ namespace cryptonote { std::string status; uint64_t fee; + uint64_t quantization_mask; bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(fee) + KV_SERIALIZE_OPT(quantization_mask, (uint64_t)1) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 55858cc2a..9d3b09b68 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -86,53 +86,45 @@ namespace rpc res.error_details = "incorrect number of transactions retrieved for block"; return; } - std::vector<transaction> txs; - for (const auto& p : it->second) - { - txs.resize(txs.size() + 1); - if (!parse_and_validate_tx_from_blob(p.second, txs.back())) - { - res.blocks.clear(); - res.output_indices.clear(); - res.status = Message::STATUS_FAILED; - res.error_details = "failed retrieving a requested transaction"; - return; - } - } cryptonote::rpc::block_output_indices& indices = res.output_indices[block_count]; // miner tx output indices { cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); } - // assume each block returned is returned with all its transactions - // in the correct order. - auto tx_it = txs.begin(); - for (const crypto::hash& h : bwt.block.tx_hashes) + auto hash_it = bwt.block.tx_hashes.begin(); + bwt.transactions.reserve(it->second.size()); + for (const auto& blob : it->second) { - bwt.transactions.emplace(h, *tx_it); - tx_it++; + bwt.transactions.emplace_back(); + if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back())) + { + res.blocks.clear(); + res.output_indices.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "failed retrieving a requested transaction"; + return; + } cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(h, tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(*hash_it, tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); + ++hash_it; } it++; @@ -268,44 +260,6 @@ namespace rpc } - //TODO: handle "restricted" RPC - void DaemonHandler::handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res) - { - auto& chain = m_core.get_blockchain_storage(); - - try - { - for (const uint64_t& amount : req.amounts) - { - std::vector<uint64_t> indices = chain.get_random_outputs(amount, req.count); - - outputs_for_amount ofa; - - ofa.resize(indices.size()); - - for (size_t i = 0; i < indices.size(); i++) - { - crypto::public_key key = chain.get_output_key(amount, indices[i]); - ofa[i].amount_index = indices[i]; - ofa[i].key = key; - } - - amount_with_random_outputs amt; - amt.amount = amount; - amt.outputs = ofa; - - res.amounts_with_outputs.push_back(amt); - } - - res.status = Message::STATUS_OK; - } - catch (const std::exception& e) - { - res.status = Message::STATUS_FAILED; - res.error_details = e.what(); - } - } - void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) { auto tx_blob = cryptonote::tx_to_blob(req.tx); @@ -480,7 +434,8 @@ namespace rpc res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); - res.info.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); + res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); res.status = Message::STATUS_OK; @@ -771,7 +726,7 @@ namespace rpc void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res) { - res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.num_grace_blocks); + res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); res.status = Message::STATUS_OK; } @@ -831,7 +786,6 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetTransactions, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, KeyImagesSpent, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetTxGlobalOutputIndices, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetRandomOutputsForAmounts, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, SendRawTx, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetInfo, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StartMining, req_json, resp_message, handle); diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index f43711640..5f9687511 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -66,8 +66,6 @@ class DaemonHandler : public RpcHandler void handle(const GetTxGlobalOutputIndices::Request& req, GetTxGlobalOutputIndices::Response& res); - void handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res); - void handle(const SendRawTx::Request& req, SendRawTx::Response& res); void handle(const StartMining::Request& req, StartMining::Response& res); diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index e5fb9781c..56f6f6a8c 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -41,7 +41,6 @@ const char* const GetHashesFast::name = "get_hashes_fast"; const char* const GetTransactions::name = "get_transactions"; const char* const KeyImagesSpent::name = "key_images_spent"; const char* const GetTxGlobalOutputIndices::name = "get_tx_global_output_indices"; -const char* const GetRandomOutputsForAmounts::name = "get_random_outputs_for_amounts"; const char* const SendRawTx::name = "send_raw_tx"; const char* const StartMining::name = "start_mining"; const char* const StopMining::name = "stop_mining"; @@ -273,42 +272,6 @@ void GetTxGlobalOutputIndices::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, output_indices, output_indices); } - -rapidjson::Value GetRandomOutputsForAmounts::Request::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); - INSERT_INTO_JSON_OBJECT(val, doc, count, count); - - return val; -} - -void GetRandomOutputsForAmounts::Request::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts, amounts); - GET_FROM_JSON_OBJECT(val, count, count); -} - -rapidjson::Value GetRandomOutputsForAmounts::Response::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts_with_outputs, amounts_with_outputs); - - return val; -} - -void GetRandomOutputsForAmounts::Response::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts_with_outputs, amounts_with_outputs); -} - - rapidjson::Value SendRawTx::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index fc1b2329d..cf15ade1c 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -44,7 +44,7 @@ namespace rpc struct block_with_transactions { cryptonote::block block; - std::unordered_map<crypto::hash, cryptonote::transaction> transactions; + std::vector<cryptonote::transaction> transactions; }; typedef std::vector<uint64_t> tx_output_indices; @@ -85,6 +85,7 @@ namespace rpc cryptonote::transaction tx; crypto::hash tx_hash; uint64_t blob_size; + uint64_t weight; uint64_t fee; crypto::hash max_used_block_hash; uint64_t max_used_block_height; @@ -185,6 +186,9 @@ namespace rpc crypto::hash top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; + uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; }; diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index d4044d11b..60c78480a 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -82,11 +82,17 @@ namespace cryptonote } } - if (command_line::has_arg(vm, arg.rpc_login)) + const char *env_rpc_login = nullptr; + const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login); + const bool use_rpc_env = !has_rpc_arg && (env_rpc_login = getenv("RPC_LOGIN")) != nullptr && strlen(env_rpc_login) > 0; + boost::optional<tools::login> login{}; + if (has_rpc_arg || use_rpc_env) { - config.login = tools::login::parse(command_line::get_arg(vm, arg.rpc_login), true, [](bool verify) { - return tools::password_container::prompt(verify, "RPC server password"); - }); + config.login = tools::login::parse( + has_rpc_arg ? command_line::get_arg(vm, arg.rpc_login) : std::string(env_rpc_login), true, [](bool verify) { + return tools::password_container::prompt(verify, "RPC server password"); + }); + if (!config.login) return boost::none; diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index c2467b863..89a1dbd23 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -28,6 +28,8 @@ #include "json_object.h" +#include <boost/range/adaptor/transformed.hpp> +#include <boost/variant/apply_visitor.hpp> #include <limits> #include <type_traits> #include "string_tools.h" @@ -219,11 +221,11 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::transaction& tx, ra INSERT_INTO_JSON_OBJECT(val, doc, version, tx.version); INSERT_INTO_JSON_OBJECT(val, doc, unlock_time, tx.unlock_time); - INSERT_INTO_JSON_OBJECT(val, doc, vin, tx.vin); - INSERT_INTO_JSON_OBJECT(val, doc, vout, tx.vout); + INSERT_INTO_JSON_OBJECT(val, doc, inputs, tx.vin); + INSERT_INTO_JSON_OBJECT(val, doc, outputs, tx.vout); INSERT_INTO_JSON_OBJECT(val, doc, extra, tx.extra); INSERT_INTO_JSON_OBJECT(val, doc, signatures, tx.signatures); - INSERT_INTO_JSON_OBJECT(val, doc, rct_signatures, tx.rct_signatures); + INSERT_INTO_JSON_OBJECT(val, doc, ringct, tx.rct_signatures); } @@ -236,11 +238,11 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx) GET_FROM_JSON_OBJECT(val, tx.version, version); GET_FROM_JSON_OBJECT(val, tx.unlock_time, unlock_time); - GET_FROM_JSON_OBJECT(val, tx.vin, vin); - GET_FROM_JSON_OBJECT(val, tx.vout, vout); + GET_FROM_JSON_OBJECT(val, tx.vin, inputs); + GET_FROM_JSON_OBJECT(val, tx.vout, outputs); GET_FROM_JSON_OBJECT(val, tx.extra, extra); GET_FROM_JSON_OBJECT(val, tx.signatures, signatures); - GET_FROM_JSON_OBJECT(val, tx.rct_signatures, rct_signatures); + GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct); } void toJsonValue(rapidjson::Document& doc, const cryptonote::block& b, rapidjson::Value& val) @@ -277,26 +279,31 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::txin_v& txin, rapid { val.SetObject(); - if (txin.type() == typeid(cryptonote::txin_gen)) + struct add_input { - val.AddMember("type", "txin_gen", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_gen>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_script)) - { - val.AddMember("type", "txin_to_script", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_script>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_scripthash)) - { - val.AddMember("type", "txin_to_scripthash", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_scripthash>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_key)) - { - val.AddMember("type", "txin_to_key", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_key>(txin)); - } + using result_type = void; + + rapidjson::Document& doc; + rapidjson::Value& val; + + void operator()(cryptonote::txin_to_key const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_key, input); + } + void operator()(cryptonote::txin_gen const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, gen, input); + } + void operator()(cryptonote::txin_to_script const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_script, input); + } + void operator()(cryptonote::txin_to_scripthash const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_scripthash, input); + } + }; + boost::apply_visitor(add_input{doc, val}, txin); } @@ -307,31 +314,37 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin) throw WRONG_TYPE("json object"); } - OBJECT_HAS_MEMBER_OR_THROW(val, "type") - OBJECT_HAS_MEMBER_OR_THROW(val, "value") - if (val["type"]== "txin_gen") - { - cryptonote::txin_gen tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; - } - else if (val["type"]== "txin_to_script") + if (val.MemberCount() != 1) { - cryptonote::txin_to_script tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; + throw MISSING_KEY("Invalid input object"); } - else if (val["type"] == "txin_to_scripthash") - { - cryptonote::txin_to_scripthash tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; - } - else if (val["type"] == "txin_to_key") + + for (auto const& elem : val.GetObject()) { - cryptonote::txin_to_key tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; + if (elem.name == "to_key") + { + cryptonote::txin_to_key tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "gen") + { + cryptonote::txin_gen tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "to_script") + { + cryptonote::txin_to_script tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "to_scripthash") + { + cryptonote::txin_to_scripthash tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } } } @@ -405,7 +418,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::txin_to_key& txin, INSERT_INTO_JSON_OBJECT(val, doc, amount, txin.amount); INSERT_INTO_JSON_OBJECT(val, doc, key_offsets, txin.key_offsets); - INSERT_INTO_JSON_OBJECT(val, doc, k_image, txin.k_image); + INSERT_INTO_JSON_OBJECT(val, doc, key_image, txin.k_image); } @@ -418,58 +431,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin) GET_FROM_JSON_OBJECT(val, txin.amount, amount); GET_FROM_JSON_OBJECT(val, txin.key_offsets, key_offsets); - GET_FROM_JSON_OBJECT(val, txin.k_image, k_image); -} - -void toJsonValue(rapidjson::Document& doc, const cryptonote::txout_target_v& txout, rapidjson::Value& val) -{ - val.SetObject(); - - if (txout.type() == typeid(cryptonote::txout_to_script)) - { - val.AddMember("type", "txout_to_script", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_script>(txout)); - } - else if (txout.type() == typeid(cryptonote::txout_to_scripthash)) - { - val.AddMember("type", "txout_to_scripthash", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_scripthash>(txout)); - } - else if (txout.type() == typeid(cryptonote::txout_to_key)) - { - val.AddMember("type", "txout_to_key", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_key>(txout)); - } -} - - -void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_target_v& txout) -{ - if (!val.IsObject()) - { - throw WRONG_TYPE("json object"); - } - - OBJECT_HAS_MEMBER_OR_THROW(val, "type") - OBJECT_HAS_MEMBER_OR_THROW(val, "value") - if (val["type"]== "txout_to_script") - { - cryptonote::txout_to_script tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } - else if (val["type"] == "txout_to_scripthash") - { - cryptonote::txout_to_scripthash tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } - else if (val["type"] == "txout_to_key") - { - cryptonote::txout_to_key tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } + GET_FROM_JSON_OBJECT(val, txin.k_image, key_image); } void toJsonValue(rapidjson::Document& doc, const cryptonote::txout_to_script& txout, rapidjson::Value& val) @@ -533,7 +495,28 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::tx_out& txout, rapi val.SetObject(); INSERT_INTO_JSON_OBJECT(val, doc, amount, txout.amount); - INSERT_INTO_JSON_OBJECT(val, doc, target, txout.target); + + struct add_output + { + using result_type = void; + + rapidjson::Document& doc; + rapidjson::Value& val; + + void operator()(cryptonote::txout_to_key const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_key, output); + } + void operator()(cryptonote::txout_to_script const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_script, output); + } + void operator()(cryptonote::txout_to_scripthash const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_scripthash, output); + } + }; + boost::apply_visitor(add_output{doc, val}, txout.target); } void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout) @@ -543,8 +526,37 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout) throw WRONG_TYPE("json object"); } - GET_FROM_JSON_OBJECT(val, txout.amount, amount); - GET_FROM_JSON_OBJECT(val, txout.target, target); + if (val.MemberCount() != 2) + { + throw MISSING_KEY("Invalid input object"); + } + + for (auto const& elem : val.GetObject()) + { + if (elem.name == "amount") + { + fromJsonValue(elem.value, txout.amount); + } + + if (elem.name == "to_key") + { + cryptonote::txout_to_key tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + else if (elem.name == "to_script") + { + cryptonote::txout_to_script tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + else if (elem.name == "to_scripthash") + { + cryptonote::txout_to_scripthash tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + } } void toJsonValue(rapidjson::Document& doc, const cryptonote::connection_info& info, rapidjson::Value& val) @@ -617,7 +629,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::block_complete_entr val.SetObject(); INSERT_INTO_JSON_OBJECT(val, doc, block, blk.block); - INSERT_INTO_JSON_OBJECT(val, doc, txs, blk.txs); + INSERT_INTO_JSON_OBJECT(val, doc, transactions, blk.txs); } @@ -629,7 +641,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry } GET_FROM_JSON_OBJECT(val, blk.block, block); - GET_FROM_JSON_OBJECT(val, blk.txs, txs); + GET_FROM_JSON_OBJECT(val, blk.txs, transactions); } void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::block_with_transactions& blk, rapidjson::Value& val) @@ -745,6 +757,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx INSERT_INTO_JSON_OBJECT(val, doc, tx, tx.tx); INSERT_INTO_JSON_OBJECT(val, doc, tx_hash, tx.tx_hash); INSERT_INTO_JSON_OBJECT(val, doc, blob_size, tx.blob_size); + INSERT_INTO_JSON_OBJECT(val, doc, weight, tx.weight); INSERT_INTO_JSON_OBJECT(val, doc, fee, tx.fee); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_hash, tx.max_used_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_height, tx.max_used_block_height); @@ -768,6 +781,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx) GET_FROM_JSON_OBJECT(val, tx.tx, tx); GET_FROM_JSON_OBJECT(val, tx.blob_size, blob_size); + GET_FROM_JSON_OBJECT(val, tx.weight, weight); GET_FROM_JSON_OBJECT(val, tx.fee, fee); GET_FROM_JSON_OBJECT(val, tx.max_used_block_hash, max_used_block_hash); GET_FROM_JSON_OBJECT(val, tx.max_used_block_height, max_used_block_height); @@ -936,51 +950,70 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp void toJsonValue(rapidjson::Document& doc, const rct::rctSig& sig, rapidjson::Value& val) { + using boost::adaptors::transform; + val.SetObject(); + const auto just_mask = [] (rct::ctkey const& key) -> rct::key const& + { + return key.mask; + }; + INSERT_INTO_JSON_OBJECT(val, doc, type, sig.type); - INSERT_INTO_JSON_OBJECT(val, doc, message, sig.message); - INSERT_INTO_JSON_OBJECT(val, doc, mixRing, sig.mixRing); - INSERT_INTO_JSON_OBJECT(val, doc, pseudoOuts, sig.pseudoOuts); - INSERT_INTO_JSON_OBJECT(val, doc, ecdhInfo, sig.ecdhInfo); - INSERT_INTO_JSON_OBJECT(val, doc, outPk, sig.outPk); - INSERT_INTO_JSON_OBJECT(val, doc, txnFee, sig.txnFee); - INSERT_INTO_JSON_OBJECT(val, doc, p, sig.p); + INSERT_INTO_JSON_OBJECT(val, doc, encrypted, sig.ecdhInfo); + INSERT_INTO_JSON_OBJECT(val, doc, commitments, transform(sig.outPk, just_mask)); + INSERT_INTO_JSON_OBJECT(val, doc, fee, sig.txnFee); + + // prunable + { + rapidjson::Value prunable; + prunable.SetObject(); + + INSERT_INTO_JSON_OBJECT(prunable, doc, range_proofs, sig.p.rangeSigs); + INSERT_INTO_JSON_OBJECT(prunable, doc, bulletproofs, sig.p.bulletproofs); + INSERT_INTO_JSON_OBJECT(prunable, doc, mlsags, sig.p.MGs); + INSERT_INTO_JSON_OBJECT(prunable, doc, pseudo_outs, sig.get_pseudo_outs()); + + val.AddMember("prunable", prunable, doc.GetAllocator()); + } } void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig) { + using boost::adaptors::transform; + if (!val.IsObject()) { throw WRONG_TYPE("json object"); } + std::vector<rct::key> commitments; + GET_FROM_JSON_OBJECT(val, sig.type, type); - GET_FROM_JSON_OBJECT(val, sig.message, message); - GET_FROM_JSON_OBJECT(val, sig.mixRing, mixRing); - GET_FROM_JSON_OBJECT(val, sig.pseudoOuts, pseudoOuts); - GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, ecdhInfo); - GET_FROM_JSON_OBJECT(val, sig.outPk, outPk); - GET_FROM_JSON_OBJECT(val, sig.txnFee, txnFee); - GET_FROM_JSON_OBJECT(val, sig.p, p); -} + GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted); + GET_FROM_JSON_OBJECT(val, commitments, commitments); + GET_FROM_JSON_OBJECT(val, sig.txnFee, fee); -void toJsonValue(rapidjson::Document& doc, const rct::ctkey& key, rapidjson::Value& val) -{ - val.SetObject(); + // prunable + { + OBJECT_HAS_MEMBER_OR_THROW(val, "prunable"); + const auto& prunable = val["prunable"]; - INSERT_INTO_JSON_OBJECT(val, doc, dest, key.dest); - INSERT_INTO_JSON_OBJECT(val, doc, mask, key.mask); -} + rct::keyV pseudo_outs; -void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key) -{ - if (!val.IsObject()) + GET_FROM_JSON_OBJECT(prunable, sig.p.rangeSigs, range_proofs); + GET_FROM_JSON_OBJECT(prunable, sig.p.bulletproofs, bulletproofs); + GET_FROM_JSON_OBJECT(prunable, sig.p.MGs, mlsags); + GET_FROM_JSON_OBJECT(prunable, pseudo_outs, pseudo_outs); + + sig.get_pseudo_outs() = std::move(pseudo_outs); + } + + sig.outPk.reserve(commitments.size()); + for (rct::key const& commitment : commitments) { - throw WRONG_TYPE("json object"); + sig.outPk.push_back({{}, commitment}); } - GET_FROM_JSON_OBJECT(val, key.dest, dest); - GET_FROM_JSON_OBJECT(val, key.mask, mask); } void toJsonValue(rapidjson::Document& doc, const rct::ecdhTuple& tuple, rapidjson::Value& val) @@ -1002,27 +1035,6 @@ void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple) GET_FROM_JSON_OBJECT(val, tuple.amount, amount); } -void toJsonValue(rapidjson::Document& doc, const rct::rctSigPrunable& sig, rapidjson::Value& val) -{ - val.SetObject(); - - INSERT_INTO_JSON_OBJECT(val, doc, rangeSigs, sig.rangeSigs); - INSERT_INTO_JSON_OBJECT(val, doc, bulletproofs, sig.bulletproofs); - INSERT_INTO_JSON_OBJECT(val, doc, MGs, sig.MGs); -} - -void fromJsonValue(const rapidjson::Value& val, rct::rctSigPrunable& sig) -{ - if (!val.IsObject()) - { - throw WRONG_TYPE("json object"); - } - - GET_FROM_JSON_OBJECT(val, sig.rangeSigs, rangeSigs); - GET_FROM_JSON_OBJECT(val, sig.bulletproofs, bulletproofs); - GET_FROM_JSON_OBJECT(val, sig.MGs, MGs); -} - void toJsonValue(rapidjson::Document& doc, const rct::rangeSig& sig, rapidjson::Value& val) { val.SetObject(); @@ -1040,10 +1052,16 @@ void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig) throw WRONG_TYPE("json object"); } + const auto ci = val.FindMember("Ci"); + if (ci == val.MemberEnd()) + { + throw MISSING_KEY("Ci"); + } + GET_FROM_JSON_OBJECT(val, sig.asig, asig); std::vector<rct::key> keyVector; - cryptonote::json::fromJsonValue(val["Ci"], keyVector); + cryptonote::json::fromJsonValue(ci->value, keyVector); if (!(keyVector.size() == 64)) { throw WRONG_TYPE("key64 (rct::key[64])"); @@ -1098,10 +1116,10 @@ void toJsonValue(rapidjson::Document& doc, const rct::boroSig& sig, rapidjson::V val.SetObject(); std::vector<rct::key> keyVector(sig.s0, std::end(sig.s0)); - INSERT_INTO_JSON_OBJECT(val, doc, s0, sig.s0); + INSERT_INTO_JSON_OBJECT(val, doc, s0, keyVector); keyVector.assign(sig.s1, std::end(sig.s1)); - INSERT_INTO_JSON_OBJECT(val, doc, s1, sig.s1); + INSERT_INTO_JSON_OBJECT(val, doc, s1, keyVector); INSERT_INTO_JSON_OBJECT(val, doc, ee, sig.ee); } @@ -1179,6 +1197,9 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); INSERT_INTO_JSON_OBJECT(val, doc, block_size_limit, info.block_size_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_limit, info.block_weight_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_size_median, info.block_size_median); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_median, info.block_weight_median); INSERT_INTO_JSON_OBJECT(val, doc, start_time, info.start_time); } @@ -1205,6 +1226,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); GET_FROM_JSON_OBJECT(val, info.block_size_limit, block_size_limit); + GET_FROM_JSON_OBJECT(val, info.block_weight_limit, block_weight_limit); + GET_FROM_JSON_OBJECT(val, info.block_size_median, block_size_median); + GET_FROM_JSON_OBJECT(val, info.block_weight_median, block_weight_median); GET_FROM_JSON_OBJECT(val, info.start_time, start_time); } diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index de21ace66..da3351fe3 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -263,15 +263,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp void toJsonValue(rapidjson::Document& doc, const rct::rctSig& i, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& i, rct::rctSig& sig); -void toJsonValue(rapidjson::Document& doc, const rct::ctkey& key, rapidjson::Value& val); -void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key); - void toJsonValue(rapidjson::Document& doc, const rct::ecdhTuple& tuple, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple); -void toJsonValue(rapidjson::Document& doc, const rct::rctSigPrunable& sig, rapidjson::Value& val); -void fromJsonValue(const rapidjson::Value& val, rct::rctSigPrunable& sig); - void toJsonValue(rapidjson::Document& doc, const rct::rangeSig& sig, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 193b62571..391d1f03e 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -86,26 +86,27 @@ typedef cryptonote::simple_wallet sw; #define EXTENDED_LOGS_FILE "wallet_details.log" -#define DEFAULT_MIX 6 +#define DEFAULT_MIX 10 -#define MIN_RING_SIZE 7 // Used to inform user about min ring size -- does not track actual protocol +#define MIN_RING_SIZE 11 // Used to inform user about min ring size -- does not track actual protocol #define LOCK_IDLE_SCOPE() \ bool auto_refresh_enabled = m_auto_refresh_enabled.load(std::memory_order_relaxed); \ m_auto_refresh_enabled.store(false, std::memory_order_relaxed); \ /* stop any background refresh, and take over */ \ m_wallet->stop(); \ - m_idle_mutex.lock(); \ - while (m_auto_refresh_refreshing) \ - m_idle_cond.notify_one(); \ - m_idle_mutex.unlock(); \ -/* if (auto_refresh_run)*/ \ - /*m_auto_refresh_thread.join();*/ \ boost::unique_lock<boost::mutex> lock(m_idle_mutex); \ + m_idle_cond.notify_all(); \ epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \ m_auto_refresh_enabled.store(auto_refresh_enabled, std::memory_order_relaxed); \ }) +#define SCOPED_WALLET_UNLOCK() \ + LOCK_IDLE_SCOPE(); \ + boost::optional<tools::password_container> pwd_container = boost::none; \ + if (m_wallet->ask_password() && !m_wallet->watch_only() && !(pwd_container = get_and_verify_password())) { return true; } \ + tools::wallet_keys_unlocker unlocker(*m_wallet, pwd_container); + enum TransferType { TransferOriginal, TransferNew, @@ -128,8 +129,6 @@ namespace const command_line::arg_descriptor<bool> arg_restore_deterministic_wallet = {"restore-deterministic-wallet", sw::tr("Recover wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_restore_multisig_wallet = {"restore-multisig-wallet", sw::tr("Recover multisig wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Generate non-deterministic view and spend keys"), false}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false}; - const command_line::arg_descriptor<bool> arg_untrusted_daemon = {"untrusted-daemon", sw::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> arg_allow_mismatched_daemon_version = {"allow-mismatched-daemon-version", sw::tr("Allow communicating with a daemon that uses a different RPC version"), false}; const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; const command_line::arg_descriptor<bool> arg_do_not_relay = {"do-not-relay", sw::tr("The newly created transaction will not be relayed to the monero network"), false}; @@ -152,12 +151,31 @@ namespace // // Note also that input for passwords is NOT translated, to remain compatible with any // passwords containing special characters that predate this switch to UTF-8 support. - static std::string cp850_to_utf8(const std::string &cp850_str) + template<typename T> + static T cp850_to_utf8(const T &cp850_str) { boost::locale::generator gen; gen.locale_cache_enabled(true); std::locale loc = gen("en_US.CP850"); - return boost::locale::conv::to_utf<char>(cp850_str, loc); + const boost::locale::conv::method_type how = boost::locale::conv::default_method; + T result; + const char *begin = cp850_str.data(); + const char *end = begin + cp850_str.size(); + result.reserve(end-begin); + typedef std::back_insert_iterator<T> inserter_type; + inserter_type inserter(result); + boost::locale::utf::code_point c; + while(begin!=end) { + c=boost::locale::utf::utf_traits<char>::template decode<char const *>(begin,end); + if(c==boost::locale::utf::illegal || c==boost::locale::utf::incomplete) { + if(how==boost::locale::conv::stop) + throw boost::locale::conv::conversion_error(); + } + else { + boost::locale::utf::utf_traits<char>::template encode<inserter_type>(c,inserter); + } + } + return result; } #endif @@ -177,6 +195,28 @@ namespace return epee::string_tools::trim(buf); } + epee::wipeable_string input_secure_line(const std::string& prompt) + { +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + auto pwd_container = tools::password_container::prompt(false, prompt.c_str(), false); + if (!pwd_container) + { + MERROR("Failed to read secure line"); + return ""; + } + + epee::wipeable_string buf = pwd_container->password(); + +#ifdef WIN32 + buf = cp850_to_utf8(buf); +#endif + + buf.trim(); + return buf; + } + boost::optional<tools::password_container> password_prompter(const char *prompt, bool verify) { #ifdef HAVE_READLINE @@ -405,7 +445,7 @@ namespace LOG_ERROR("RPC error: " << e.to_string()); fail_msg_writer() << tr("RPC error: ") << e.what(); } - catch (const tools::error::get_random_outs_error &e) + catch (const tools::error::get_outs_error &e) { fail_msg_writer() << tr("failed to get random outputs to mix: ") << e.what(); } @@ -512,6 +552,18 @@ namespace } return true; } + + void print_secret_key(const crypto::secret_key &k) + { + static constexpr const char hex[] = u8"0123456789abcdef"; + const uint8_t *ptr = (const uint8_t*)k.data; + for (size_t i = 0, sz = sizeof(k); i < sz; ++i) + { + putchar(hex[*ptr >> 4]); + putchar(hex[*ptr & 15]); + ++ptr; + } + } } bool parse_priority(const std::string& arg, uint32_t& priority) @@ -527,6 +579,18 @@ bool parse_priority(const std::string& arg, uint32_t& priority) return false; } +std::string join_priority_strings(const char *delimiter) +{ + std::string s; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) + { + if (!s.empty()) + s += delimiter; + s += allowed_priority_strings[n]; + } + return s; +} + std::string simple_wallet::get_commands_str() { std::stringstream ss; @@ -561,12 +625,15 @@ std::string simple_wallet::get_command_usage(const std::vector<std::string> &arg bool simple_wallet::viewkey(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); // don't log + PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { - std::cout << "secret: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key) << std::endl; + printf("secret: "); + print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key); + putchar('\n'); } std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_view_public_key) << std::endl; @@ -580,12 +647,15 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto fail_msg_writer() << tr("wallet is watch-only and has no spend key"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); // don't log + PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { - std::cout << "secret: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_spend_secret_key) << std::endl; + printf("secret: "); + print_secret_key(m_wallet->get_account().get_keys().m_spend_secret_key); + putchar('\n'); } std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_spend_public_key) << std::endl; @@ -595,7 +665,7 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto bool simple_wallet::print_seed(bool encrypted) { bool success = false; - std::string seed; + epee::wipeable_string seed; bool ready, multisig; if (m_wallet->key_on_device()) @@ -608,7 +678,8 @@ bool simple_wallet::print_seed(bool encrypted) fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); multisig = m_wallet->multisig(&ready); if (multisig) @@ -628,7 +699,7 @@ bool simple_wallet::print_seed(bool encrypted) epee::wipeable_string seed_pass; if (encrypted) { - auto pwd_container = password_prompter(tr("Enter optional seed encryption passphrase, empty to see raw seed"), true); + auto pwd_container = password_prompter(tr("Enter optional seed offset passphrase, empty to see raw seed"), true); if (std::cin.eof() || !pwd_container) return true; seed_pass = pwd_container->password(); @@ -677,22 +748,36 @@ bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = s fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } - if (!m_wallet->is_deterministic()) - { - fail_msg_writer() << tr("wallet is non-deterministic and has no seed"); - return true; - } - - const auto pwd_container = get_and_verify_password(); - if (pwd_container) + + epee::wipeable_string password; { - std::string mnemonic_language = get_mnemonic_language(); - if (mnemonic_language.empty()) + SCOPED_WALLET_UNLOCK(); + + if (!m_wallet->is_deterministic()) + { + fail_msg_writer() << tr("wallet is non-deterministic and has no seed"); return true; + } - m_wallet->set_seed_language(std::move(mnemonic_language)); - m_wallet->rewrite(m_wallet_file, pwd_container->password()); + // we need the password, even if ask-password is unset + if (!pwd_container) + { + pwd_container = get_and_verify_password(); + if (pwd_container == boost::none) + { + fail_msg_writer() << tr("Incorrect password"); + return true; + } + } + password = pwd_container->password(); } + + std::string mnemonic_language = get_mnemonic_language(); + if (mnemonic_language.empty()) + return true; + + m_wallet->set_seed_language(std::move(mnemonic_language)); + m_wallet->rewrite(m_wallet_file, password); return true; } @@ -713,8 +798,7 @@ bool simple_wallet::change_password(const std::vector<std::string> &args) try { - m_wallet->rewrite(m_wallet_file, pwd_container->password()); - m_wallet->store(); + m_wallet->change_password(m_wallet_file, orig_pwd_container->password(), pwd_container->password()); } catch (const tools::error::wallet_logic_error& e) { @@ -745,21 +829,24 @@ bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std: fail_msg_writer() << tr("Cannot connect to daemon"); return true; } - const uint64_t per_kb_fee = m_wallet->get_per_kb_fee(); - const uint64_t typical_size_kb = 13; - message_writer() << (boost::format(tr("Current fee is %s %s per kB")) % print_money(per_kb_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point())).str(); + const bool per_byte = m_wallet->use_fork_rules(HF_VERSION_PER_BYTE_FEE); + const uint64_t base_fee = m_wallet->get_base_fee(); + const char *base = per_byte ? "byte" : "kB"; + const uint64_t typical_size = per_byte ? 2500 : 13; + const uint64_t size_granularity = per_byte ? 1 : 1024; + message_writer() << (boost::format(tr("Current fee is %s %s per %s")) % print_money(base_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % base).str(); std::vector<uint64_t> fees; for (uint32_t priority = 1; priority <= 4; ++priority) { uint64_t mult = m_wallet->get_fee_multiplier(priority); - fees.push_back(per_kb_fee * typical_size_kb * mult); + fees.push_back(base_fee * typical_size * mult); } std::vector<std::pair<uint64_t, uint64_t>> blocks; try { - uint64_t base_size = typical_size_kb * 1024; - blocks = m_wallet->estimate_backlog(base_size, base_size + 1023, fees); + uint64_t base_size = typical_size * size_granularity; + blocks = m_wallet->estimate_backlog(base_size, base_size + size_granularity - 1, fees); } catch (const std::exception &e) { @@ -817,12 +904,7 @@ bool simple_wallet::prepare_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your password is incorrect."); - return true; - } + SCOPED_WALLET_UNLOCK(); std::string multisig_info = m_wallet->get_multisig_info(); success_msg_writer() << multisig_info; @@ -855,13 +937,6 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your original password was incorrect."); - return true; - } - if (args.size() < 2) { fail_msg_writer() << tr("usage: make_multisig <threshold> <multisiginfo1> [<multisiginfo2>...]"); @@ -876,6 +951,13 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) return true; } + const auto orig_pwd_container = get_and_verify_password(); + if(orig_pwd_container == boost::none) + { + fail_msg_writer() << tr("Your original password was incorrect."); + return true; + } + LOCK_IDLE_SCOPE(); try @@ -917,6 +999,14 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + + const auto pwd_container = get_and_verify_password(); + if(pwd_container == boost::none) + { + fail_msg_writer() << tr("Your original password was incorrect."); + return true; + } + if (!m_wallet->multisig(&ready)) { fail_msg_writer() << tr("This wallet is not multisig"); @@ -928,12 +1018,7 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your original password was incorrect."); - return true; - } + LOCK_IDLE_SCOPE(); if (args.size() < 2) { @@ -943,7 +1028,7 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) try { - if (!m_wallet->finalize_multisig(orig_pwd_container->password(), args)) + if (!m_wallet->finalize_multisig(pwd_container->password(), args)) { fail_msg_writer() << tr("Failed to finalize multisig"); return true; @@ -981,8 +1066,8 @@ bool simple_wallet::export_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_multisig_info <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) - return true; + + SCOPED_WALLET_UNLOCK(); const std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) @@ -1033,8 +1118,8 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: import_multisig_info <filename1> [<filename2>...] - one for each other participant"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) - return true; + + SCOPED_WALLET_UNLOCK(); std::vector<cryptonote::blobdata> info; for (size_t n = 0; n < args.size(); ++n) @@ -1050,11 +1135,11 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) info.push_back(std::move(data)); } - LOCK_IDLE_SCOPE(); - // all read and parsed, actually import try { + m_in_manual_refresh.store(true, std::memory_order_relaxed); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); size_t n_outputs = m_wallet->import_multisig(info); // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -1065,7 +1150,7 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("Failed to import multisig info: ") << e.what(); return true; } - if (is_daemon_trusted()) + if (m_wallet->is_trusted_daemon()) { try { @@ -1112,7 +1197,8 @@ bool simple_wallet::sign_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: sign_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; std::vector<crypto::hash> txids; @@ -1185,7 +1271,8 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: submit_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); if (!try_connect_to_daemon()) return true; @@ -1217,7 +1304,7 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -1252,7 +1339,8 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_raw_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) @@ -1539,23 +1627,23 @@ bool simple_wallet::set_ring(const std::vector<std::string> &args) bool simple_wallet::blackball(const std::vector<std::string> &args) { - crypto::public_key output; + uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets; if (args.size() == 0) { - fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]"); + fail_msg_writer() << tr("usage: blackball <amount>/<offset> | <filename> [add]"); return true; } try { - if (epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &amount, &offset) == 2) { - m_wallet->blackball_output(output); + m_wallet->blackball_output(std::make_pair(amount, offset)); } else if (epee::file_io_utils::is_file_exist(args[0])) { - std::vector<crypto::public_key> outputs; - char str[65]; + std::vector<std::pair<uint64_t, uint64_t>> outputs; + char str[256]; std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); if (f) @@ -1569,10 +1657,27 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) str[len - 1] = 0; if (!str[0]) continue; - outputs.push_back(crypto::public_key()); - if (!epee::string_tools::hex_to_pod(str, outputs.back())) + if (sscanf(str, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + fail_msg_writer() << tr("First line is not an amount"); + return true; + } + if (sscanf(str, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets--) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(str, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else { - fail_msg_writer() << tr("Invalid public key: ") << str; + fail_msg_writer() << tr("Invalid output: ") << str; return true; } } @@ -1597,7 +1702,7 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) } else { - fail_msg_writer() << tr("Invalid public key, and file doesn't exist"); + fail_msg_writer() << tr("Invalid output key, and file doesn't exist"); return true; } } @@ -1611,16 +1716,16 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) bool simple_wallet::unblackball(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: unblackball <output_public_key>"); + fail_msg_writer() << tr("usage: unblackball <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } @@ -1638,25 +1743,25 @@ bool simple_wallet::unblackball(const std::vector<std::string> &args) bool simple_wallet::blackballed(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: blackballed <output_public_key>"); + fail_msg_writer() << tr("usage: blackballed <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } try { if (m_wallet->is_output_blackballed(output)) - message_writer() << tr("Blackballed: ") << output; + message_writer() << tr("Blackballed: ") << output.first << "/" << output.second; else - message_writer() << tr("not blackballed: ") << output; + message_writer() << tr("not blackballed: ") << output.first << "/" << output.second; } catch (const std::exception &e) { @@ -1754,6 +1859,8 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* if (ring_size != 0 && ring_size != DEFAULT_MIX+1) message_writer() << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); + else if (ring_size == DEFAULT_MIX) + message_writer() << tr("WARNING: from v8, ring size will be fixed and this setting will be ignored."); const auto pwd_container = get_and_verify_password(); if (pwd_container) @@ -1777,12 +1884,12 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - int priority = 0; + uint32_t priority = 0; try { if (strchr(args[1].c_str(), '-')) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4 "); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } if (args[1] == "0") @@ -1791,11 +1898,23 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } else { - priority = boost::lexical_cast<int>(args[1]); - if (priority < 1 || priority > 4) + bool found = false; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); - return true; + if (allowed_priority_strings[n] == args[1]) + { + found = true; + priority = n; + } + } + if (!found) + { + priority = boost::lexical_cast<int>(args[1]); + if (priority < 1 || priority > 4) + { + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); + return true; + } } } @@ -1809,7 +1928,7 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } catch(const boost::bad_lexical_cast &) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } catch(...) @@ -1825,6 +1944,7 @@ bool simple_wallet::set_auto_refresh(const std::vector<std::string> &args/* = st if (pwd_container) { parse_bool_and_use(args[1], [&](bool auto_refresh) { + m_auto_refresh_enabled.store(false, std::memory_order_relaxed); m_wallet->auto_refresh(auto_refresh); m_idle_mutex.lock(); m_auto_refresh_enabled.store(auto_refresh, std::memory_order_relaxed); @@ -1872,10 +1992,29 @@ bool simple_wallet::set_ask_password(const std::vector<std::string> &args/* = st const auto pwd_container = get_and_verify_password(); if (pwd_container) { - parse_bool_and_use(args[1], [&](bool r) { - m_wallet->ask_password(r); - m_wallet->rewrite(m_wallet_file, pwd_container->password()); - }); + tools::wallet2::AskPasswordType ask = tools::wallet2::AskPasswordToDecrypt; + if (args[1] == "never" || args[1] == "0") + ask = tools::wallet2::AskPasswordNever; + else if (args[1] == "action" || args[1] == "1") + ask = tools::wallet2::AskPasswordOnAction; + else if (args[1] == "encrypt" || args[1] == "decrypt" || args[1] == "2") + ask = tools::wallet2::AskPasswordToDecrypt; + else + { + fail_msg_writer() << tr("invalid argument: must be either 0/never, 1/action, or 2/encrypt/decrypt"); + return true; + } + + const tools::wallet2::AskPasswordType cur_ask = m_wallet->ask_password(); + if (!m_wallet->watch_only()) + { + if (cur_ask == tools::wallet2::AskPasswordToDecrypt && ask != tools::wallet2::AskPasswordToDecrypt) + m_wallet->decrypt_keys(pwd_container->password()); + else if (cur_ask != tools::wallet2::AskPasswordToDecrypt && ask == tools::wallet2::AskPasswordToDecrypt) + m_wallet->encrypt_keys(pwd_container->password()); + } + m_wallet->ask_password(ask); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); } return true; } @@ -2160,15 +2299,19 @@ simple_wallet::simple_wallet() tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer_original", boost::bind(&simple_wallet::transfer, this, _1), - tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer_new, this, _1), - tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), - tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <addr> <amount> <lockblocks> [<payment_id>]"), - tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"), + tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + m_cmd_binder.set_handler("locked_sweep_all", + boost::bind(&simple_wallet::locked_sweep_all, this, _1), + tr("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"), + tr("Send all unlocked balance to an address and lock it for <lockblocks> (max. 1000000). If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. <priority> is the priority of the sweep. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability.")); m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with ring_size 1")); @@ -2262,7 +2405,7 @@ simple_wallet::simple_wallet() "priority [0|1|2|3|4]\n " " Set the fee to default/unimportant/normal/elevated/priority.\n " "confirm-missing-payment-id <1|0>\n " - "ask-password <1|0>\n " + "ask-password <0|1|2 (or never|action|decrypt)>\n " "unit <monero|millinero|micronero|nanonero|piconero>\n " " Set the default monero (sub-)unit.\n " "min-outputs-count [n]\n " @@ -2298,6 +2441,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::get_tx_key, this, _1), tr("get_tx_key <txid>"), tr("Get the transaction key (r) for a given <txid>.")); + m_cmd_binder.set_handler("set_tx_key", + boost::bind(&simple_wallet::set_tx_key, this, _1), + tr("set_tx_key <txid> <tx_key>"), + tr("Set the transaction key (r) for a given <txid> in case the tx was made by some other device or 3rd party wallet.")); m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("check_tx_key <txid> <txkey> <address>"), @@ -2330,7 +2477,7 @@ simple_wallet::simple_wallet() tr("Check a signature proving that the owner of <address> holds at least this much, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("show_transfers", boost::bind(&simple_wallet::show_transfers, this, _1), - tr("show_transfers [in|out|pending|failed|pool] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"), + tr("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"), tr("Show the incoming/outgoing transfers within an optional height range.")); m_cmd_binder.set_handler("unspent_outputs", boost::bind(&simple_wallet::unspent_outputs, this, _1), @@ -2338,7 +2485,7 @@ simple_wallet::simple_wallet() tr("Show the unspent outputs of a specified address within an optional amount range.")); m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), - tr("Rescan the blockchain from scratch.")); + tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), tr("set_tx_note <txid> [free text note]"), @@ -2376,6 +2523,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::import_key_images, this, _1), tr("import_key_images <file>"), tr("Import a signed key images list and verify their spent status.")); + m_cmd_binder.set_handler("hw_reconnect", + boost::bind(&simple_wallet::hw_reconnect, this, _1), + tr("hw_reconnect"), + tr("Attempts to reconnect HW wallet.")); m_cmd_binder.set_handler("export_outputs", boost::bind(&simple_wallet::export_outputs, this, _1), tr("export_outputs <file>"), @@ -2440,15 +2591,15 @@ simple_wallet::simple_wallet() tr("Save known rings to the shared rings database")); m_cmd_binder.set_handler("blackball", boost::bind(&simple_wallet::blackball, this, _1), - tr("blackball <output public key> | <filename> [add]"), + tr("blackball <amount>/<offset> | <filename> [add]"), tr("Blackball output(s) so they never get selected as fake outputs in a ring")); m_cmd_binder.set_handler("unblackball", boost::bind(&simple_wallet::unblackball, this, _1), - tr("unblackball <output public key>"), + tr("unblackball <amount>/<offset>"), tr("Unblackballs an output so it may get selected as a fake output in a ring")); m_cmd_binder.set_handler("blackballed", boost::bind(&simple_wallet::blackballed, this, _1), - tr("blackballed <output public key>"), + tr("blackballed <amount>/<offset>"), tr("Checks whether an output is blackballed")); m_cmd_binder.set_handler("version", boost::bind(&simple_wallet::version, this, _1), @@ -2467,6 +2618,17 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) std::string seed_language = m_wallet->get_seed_language(); if (m_use_english_language_names) seed_language = crypto::ElectrumWords::get_english_name_for(seed_language); + std::string priority_string = "invalid"; + uint32_t priority = m_wallet->get_default_priority(); + if (priority < allowed_priority_strings.size()) + priority_string = allowed_priority_strings[priority]; + std::string ask_password_string = "invalid"; + switch (m_wallet->ask_password()) + { + case tools::wallet2::AskPasswordNever: ask_password_string = "never"; break; + case tools::wallet2::AskPasswordOnAction: ask_password_string = "action"; break; + case tools::wallet2::AskPasswordToDecrypt: ask_password_string = "decrypt"; break; + } success_msg_writer() << "seed = " << seed_language; success_msg_writer() << "always-confirm-transfers = " << m_wallet->always_confirm_transfers(); success_msg_writer() << "print-ring-members = " << m_wallet->print_ring_members(); @@ -2474,9 +2636,9 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "default-ring-size = " << (m_wallet->default_mixin() ? m_wallet->default_mixin() + 1 : 0); success_msg_writer() << "auto-refresh = " << m_wallet->auto_refresh(); success_msg_writer() << "refresh-type = " << get_refresh_type_name(m_wallet->get_refresh_type()); - success_msg_writer() << "priority = " << m_wallet->get_default_priority(); + success_msg_writer() << "priority = " << priority<< " (" << priority_string << ")"; success_msg_writer() << "confirm-missing-payment-id = " << m_wallet->confirm_missing_payment_id(); - success_msg_writer() << "ask-password = " << m_wallet->ask_password(); + success_msg_writer() << "ask-password = " << m_wallet->ask_password() << " (" << ask_password_string << ")"; success_msg_writer() << "unit = " << cryptonote::get_unit(cryptonote::get_default_decimal_point()); success_msg_writer() << "min-outputs-count = " << m_wallet->get_min_output_count(); success_msg_writer() << "min-outputs-value = " << cryptonote::print_money(m_wallet->get_min_output_value()); @@ -2492,6 +2654,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "subaddress-lookahead = " << lookahead.first << ":" << lookahead.second; success_msg_writer() << "segregation-height = " << m_wallet->segregation_height(); success_msg_writer() << "ignore-fractional-outputs = " << m_wallet->ignore_fractional_outputs(); + success_msg_writer() << "device_name = " << m_wallet->device_name(); return true; } else @@ -2530,9 +2693,9 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("default-ring-size", set_default_ring_size, tr("integer >= ") << MIN_RING_SIZE); CHECK_SIMPLE_VARIABLE("auto-refresh", set_auto_refresh, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("refresh-type", set_refresh_type, tr("full (slowest, no assumptions); optimize-coinbase (fast, assumes the whole coinbase is paid to a single address); no-coinbase (fastest, assumes we receive no coinbase transaction), default (same as optimize-coinbase)")); - CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4")); + CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4, or one of ") << join_priority_strings(", ")); CHECK_SIMPLE_VARIABLE("confirm-missing-payment-id", set_confirm_missing_payment_id, tr("0 or 1")); - CHECK_SIMPLE_VARIABLE("ask-password", set_ask_password, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("ask-password", set_ask_password, tr("0|1|2 (or never|action|decrypt)")); CHECK_SIMPLE_VARIABLE("unit", set_unit, tr("monero, millinero, micronero, nanonero, piconero")); CHECK_SIMPLE_VARIABLE("min-outputs-count", set_min_output_count, tr("unsigned integer")); CHECK_SIMPLE_VARIABLE("min-outputs-value", set_min_output_value, tr("amount")); @@ -2670,28 +2833,45 @@ bool simple_wallet::ask_wallet_create_if_needed() * \brief Prints the seed with a nice message * \param seed seed to print */ -void simple_wallet::print_seed(std::string seed) +void simple_wallet::print_seed(const epee::wipeable_string &seed) { success_msg_writer(true) << "\n" << tr("NOTE: the following 25 words can be used to recover access to your wallet. " "Write them down and store them somewhere safe and secure. Please do not store them in " "your email or on file storage services outside of your immediate control.\n"); - boost::replace_nth(seed, " ", 15, "\n"); - boost::replace_nth(seed, " ", 7, "\n"); // don't log - std::cout << seed << std::endl; + int space_index = 0; + size_t len = seed.size(); + for (const char *ptr = seed.data(); len--; ++ptr) + { + if (*ptr == ' ') + { + if (space_index == 15 || space_index == 7) + putchar('\n'); + else + putchar(*ptr); + ++space_index; + } + else + putchar(*ptr); + } + putchar('\n'); + fflush(stdout); } //---------------------------------------------------------------------------------------------------- -static bool might_be_partial_seed(std::string words) +static bool might_be_partial_seed(const epee::wipeable_string &words) { - std::vector<std::string> seed; + std::vector<epee::wipeable_string> seed; - boost::algorithm::trim(words); - boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); + words.split(seed); return seed.size() < 24; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::init(const boost::program_options::variables_map& vm) { + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + m_electrum_seed.wipe(); + }); + const bool testnet = tools::wallet2::has_testnet_option(vm); const bool stagenet = tools::wallet2::has_stagenet_option(vm); if (testnet && stagenet) @@ -2701,7 +2881,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; - std::string multisig_keys; + epee::wipeable_string multisig_keys; if (!handle_command_line(vm)) return false; @@ -2743,8 +2923,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { if (m_restore_multisig_wallet) { - const char *prompt = "Specify multisig seed: "; - m_electrum_seed = input_line(prompt); + const char *prompt = "Specify multisig seed"; + m_electrum_seed = input_secure_line(prompt); if (std::cin.eof()) return false; if (m_electrum_seed.empty()) @@ -2758,8 +2938,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) m_electrum_seed = ""; do { - const char *prompt = m_electrum_seed.empty() ? "Specify Electrum seed: " : "Electrum seed continued: "; - std::string electrum_seed = input_line(prompt); + const char *prompt = m_electrum_seed.empty() ? "Specify Electrum seed" : "Electrum seed continued"; + epee::wipeable_string electrum_seed = input_secure_line(prompt); if (std::cin.eof()) return false; if (electrum_seed.empty()) @@ -2767,18 +2947,21 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("specify a recovery parameter with the --electrum-seed=\"words list here\""); return false; } - m_electrum_seed += electrum_seed + " "; + m_electrum_seed += electrum_seed; + m_electrum_seed += ' '; } while (might_be_partial_seed(m_electrum_seed)); } } if (m_restore_multisig_wallet) { - if (!epee::string_tools::parse_hexstr_to_binbuff(m_electrum_seed, multisig_keys)) + const boost::optional<epee::wipeable_string> parsed = m_electrum_seed.parse_hexstr(); + if (!parsed) { fail_msg_writer() << tr("Multisig seed failed verification"); return false; } + multisig_keys = *parsed; } else { @@ -2789,7 +2972,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } } - auto pwd_container = password_prompter(tr("Enter seed encryption passphrase, empty if none"), false); + auto pwd_container = password_prompter(tr("Enter seed offset passphrase, empty if none"), false); if (std::cin.eof() || !pwd_container) return false; epee::wipeable_string seed_pass = pwd_container->password(); @@ -2800,12 +2983,13 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) crypto::secret_key key; crypto::cn_slow_hash(seed_pass.data(), seed_pass.size(), (crypto::hash&)key); sc_reduce32((unsigned char*)key.data); - multisig_keys = m_wallet->decrypt(multisig_keys, key, true); + multisig_keys = m_wallet->decrypt<epee::wipeable_string>(std::string(multisig_keys.data(), multisig_keys.size()), key, true); } else m_recovery_key = cryptonote::decrypt_key(m_recovery_key, seed_pass); } } + epee::wipeable_string password; if (!m_generate_from_view_key.empty()) { m_wallet_file = m_generate_from_view_key; @@ -2830,20 +3014,19 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - std::string viewkey_string = input_line("View key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); if (std::cin.eof()) return false; if (viewkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key viewkey; + if (viewkey_string.hex_to_pod(unwrap(unwrap(viewkey)))) { fail_msg_writer() << tr("failed to parse view key secret key"); return false; } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); m_wallet_file=m_generate_from_view_key; @@ -2858,27 +3041,29 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - bool r = new_wallet(vm, info.address, boost::none, viewkey); + auto r = new_wallet(vm, info.address, boost::none, viewkey); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } else if (!m_generate_from_spend_key.empty()) { m_wallet_file = m_generate_from_spend_key; // parse spend secret key - std::string spendkey_string = input_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); if (std::cin.eof()) return false; if (spendkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - if (!epee::string_tools::hex_to_pod(spendkey_string, m_recovery_key)) + if (!spendkey_string.hex_to_pod(unwrap(unwrap(m_recovery_key)))) { fail_msg_writer() << tr("failed to parse spend key secret key"); return false; } - bool r = new_wallet(vm, m_recovery_key, true, false, ""); + auto r = new_wallet(vm, m_recovery_key, true, false, ""); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } else if (!m_generate_from_keys.empty()) { @@ -2904,36 +3089,34 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse spend secret key - std::string spendkey_string = input_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); if (std::cin.eof()) return false; if (spendkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata spendkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data) || spendkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key spendkey; + if (!spendkey_string.hex_to_pod(unwrap(unwrap(spendkey)))) { fail_msg_writer() << tr("failed to parse spend key secret key"); return false; } - crypto::secret_key spendkey = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data()); // parse view secret key - std::string viewkey_string = input_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); if (std::cin.eof()) return false; if (viewkey_string.empty()) { fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key viewkey; + if(!viewkey_string.hex_to_pod(unwrap(unwrap(viewkey)))) { fail_msg_writer() << tr("failed to parse view key secret key"); return false; } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); m_wallet_file=m_generate_from_keys; @@ -2955,8 +3138,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("view key does not match standard address"); return false; } - bool r = new_wallet(vm, info.address, spendkey, viewkey); + auto r = new_wallet(vm, info.address, spendkey, viewkey); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } // Asks user for all the data required to merge secret keys from multisig wallets into one master wallet, which then gets full control of the multisig wallet. The resulting wallet will be the same as any other regular wallet. @@ -3008,7 +3192,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse secret view key - std::string viewkey_string = input_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); if (std::cin.eof()) return false; if (viewkey_string.empty()) @@ -3016,13 +3200,12 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + crypto::secret_key viewkey; + if(!viewkey_string.hex_to_pod(unwrap(unwrap(viewkey)))) { fail_msg_writer() << tr("failed to parse secret view key"); return false; } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); // check that the view key matches the given address crypto::public_key pkey; @@ -3043,12 +3226,12 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if(multisig_m == multisig_n) { std::vector<crypto::secret_key> multisig_secret_spendkeys(multisig_n); - std::string spendkey_string; + epee::wipeable_string spendkey_string; cryptonote::blobdata spendkey_data; // get N secret spend keys from user for(unsigned int i=0; i<multisig_n; ++i) { - spendkey_string = input_line(tr((boost::format(tr("Secret spend key (%u of %u):")) % (i+1) % multisig_m).str().c_str())); + spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u):")) % (i+1) % multisig_m).str().c_str())); if (std::cin.eof()) return false; if (spendkey_string.empty()) @@ -3056,12 +3239,11 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("No data supplied, cancelled"); return false; } - if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data) || spendkey_data.size() != sizeof(crypto::secret_key)) + if(!spendkey_string.hex_to_pod(unwrap(unwrap(multisig_secret_spendkeys[i])))) { fail_msg_writer() << tr("failed to parse spend key secret key"); return false; } - multisig_secret_spendkeys[i] = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data()); } // sum the spend keys together to get the master spend key @@ -3089,8 +3271,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // create wallet - bool r = new_wallet(vm, info.address, spendkey, viewkey); + auto r = new_wallet(vm, info.address, spendkey, viewkey); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } else if (!m_generate_from_json.empty()) @@ -3098,7 +3281,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) m_wallet_file = m_generate_from_json; try { - m_wallet = tools::wallet2::make_from_json(vm, m_wallet_file, password_prompter); + m_wallet = tools::wallet2::make_from_json(vm, false, m_wallet_file, password_prompter); } catch (const std::exception &e) { @@ -3112,8 +3295,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_device; // create wallet - bool r = new_wallet(vm, "Ledger"); + auto r = new_wallet(vm); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; // if no block_height is specified, assume its a new account and start it "now" if(m_wallet->get_refresh_from_block_height() == 0) { { @@ -3138,12 +3322,13 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } m_wallet_file = m_generate_new; - bool r; + boost::optional<epee::wipeable_string> r; if (m_restore_multisig_wallet) r = new_wallet(vm, multisig_keys, old_language); else r = new_wallet(vm, m_recovery_key, m_restore_deterministic_wallet, m_non_deterministic, old_language); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + password = *r; } if (m_restoring && m_generate_from_json.empty() && m_generate_from_device.empty()) @@ -3225,6 +3410,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } m_wallet->set_refresh_from_block_height(m_restore_height); } + m_wallet->rewrite(m_wallet_file, password); } else { @@ -3243,22 +3429,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - // set --trusted-daemon if local and not overridden - if (!m_trusted_daemon) - { - try - { - m_trusted_daemon = false; - if (tools::is_local_address(m_wallet->get_daemon_address())) - { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; - } - } - catch (const std::exception &e) { } - } - - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) message_writer() << (boost::format(tr("Warning: using an untrusted daemon at %s, privacy will be lessened")) % m_wallet->get_daemon_address()).str(); if (m_wallet->get_ring_database().empty()) @@ -3292,10 +3463,6 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet = command_line::get_arg(vm, arg_restore_deterministic_wallet); m_restore_multisig_wallet = command_line::get_arg(vm, arg_restore_multisig_wallet); m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) || !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - m_trusted_daemon = command_line::get_arg(vm, arg_trusted_daemon) && !command_line::get_arg(vm, arg_untrusted_daemon); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) && !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - message_writer() << tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted"); m_allow_mismatched_daemon_version = command_line::get_arg(vm, arg_allow_mismatched_daemon_version); m_restore_height = command_line::get_arg(vm, arg_restore_height); m_do_not_relay = command_line::get_arg(vm, arg_do_not_relay); @@ -3344,14 +3511,16 @@ bool simple_wallet::try_connect_to_daemon(bool silent, uint32_t* version) */ std::string simple_wallet::get_mnemonic_language() { - std::vector<std::string> language_list; + std::vector<std::string> language_list_self, language_list_english; + const std::vector<std::string> &language_list = m_use_english_language_names ? language_list_english : language_list_self; std::string language_choice; int language_number = -1; - crypto::ElectrumWords::get_language_list(language_list, m_use_english_language_names); + crypto::ElectrumWords::get_language_list(language_list_self, false); + crypto::ElectrumWords::get_language_list(language_list_english, true); std::cout << tr("List of available languages for your wallet's seed:") << std::endl; std::cout << tr("If your display freezes, exit blind with ^C, then run again with --use-english-language-names") << std::endl; int ii; - std::vector<std::string>::iterator it; + std::vector<std::string>::const_iterator it; for (it = language_list.begin(), ii = 0; it != language_list.end(); it++, ii++) { std::cout << ii << " : " << *it << std::endl; @@ -3375,7 +3544,7 @@ std::string simple_wallet::get_mnemonic_language() fail_msg_writer() << tr("invalid language choice entered. Please try again.\n"); } } - return language_list[language_number]; + return language_list_self[language_number]; } //---------------------------------------------------------------------------------------------------- boost::optional<tools::password_container> simple_wallet::get_and_verify_password() const @@ -3392,15 +3561,16 @@ boost::optional<tools::password_container> simple_wallet::get_and_verify_passwor return pwd_container; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, const crypto::secret_key& recovery_key, bool recover, bool two_random, const std::string &old_language) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3436,7 +3606,7 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, } mnemonic_language = get_mnemonic_language(); if (mnemonic_language.empty()) - return false; + return {}; } m_wallet->set_seed_language(mnemonic_language); @@ -3449,16 +3619,19 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, recovery_val = m_wallet->generate(m_wallet_file, std::move(rc.second).password(), recovery_key, recover, two_random, create_address_file); message_writer(console_color_white, true) << tr("Generated new wallet: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); - std::cout << tr("View key: ") << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key) << ENDL; + PAUSE_READLINE(); + std::cout << tr("View key: "); + print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key); + putchar('\n'); } catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } // convert rng value to electrum-style word list - std::string electrum_words; + epee::wipeable_string electrum_words; crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language); @@ -3479,19 +3652,20 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, } success_msg_writer() << "**********************************************************************"; - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, const cryptonote::account_public_address& address, const boost::optional<crypto::secret_key>& spendkey, const crypto::secret_key& viewkey) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3521,22 +3695,23 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, - const std::string &device_name) { - auto rc = tools::wallet2::make_new(vm, password_prompter); +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm) +{ + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3548,30 +3723,33 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, if (m_restore_height) m_wallet->set_refresh_from_block_height(m_restore_height); + auto device_desc = tools::wallet2::device_name_option(vm); try { - m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_name); + bool create_address_file = command_line::get_arg(vm, arg_create_address_file); + m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_desc.empty() ? "Ledger" : device_desc, create_address_file); message_writer(console_color_white, true) << tr("Generated new wallet on hw device: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); } catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, - const std::string &multisig_keys, const std::string &old_language) +boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, + const epee::wipeable_string &multisig_keys, const std::string &old_language) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { - return false; + return {}; } + epee::wipeable_string password = rc.second.password(); if (!m_subaddress_lookahead.empty()) { @@ -3601,7 +3779,7 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, if (!m_wallet->multisig(&ready, &threshold, &total) || !ready) { fail_msg_writer() << tr("failed to generate new mutlisig wallet"); - return false; + return {}; } message_writer(console_color_white, true) << boost::format(tr("Generated new %u/%u multisig wallet: ")) % threshold % total << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); @@ -3609,10 +3787,10 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm, catch (const std::exception& e) { fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); - return false; + return {}; } - return true; + return std::move(password); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) @@ -3636,7 +3814,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) epee::wipeable_string password; try { - auto rc = tools::wallet2::make_from_file(vm, m_wallet_file, password_prompter); + auto rc = tools::wallet2::make_from_file(vm, false, m_wallet_file, password_prompter); m_wallet = std::move(rc.first); password = std::move(std::move(rc.second).password()); if (!m_wallet) @@ -3663,7 +3841,12 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) // NOTE: this is_deprecated() refers to the wallet file format before becoming JSON. It does not refer to the "old english" seed words form of "deprecated" used elsewhere. if (m_wallet->is_deprecated()) { - if (m_wallet->is_deterministic()) + bool is_deterministic; + { + SCOPED_WALLET_UNLOCK(); + is_deterministic = m_wallet->is_deterministic(); + } + if (is_deterministic) { message_writer(console_color_green, false) << "\n" << tr("You had been using " "a deprecated version of the wallet. Please proceed to upgrade your wallet.\n"); @@ -3674,7 +3857,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) m_wallet->rewrite(m_wallet_file, password); // Display the seed - std::string seed; + epee::wipeable_string seed; m_wallet->get_seed(seed); print_seed(seed); } @@ -3793,7 +3976,7 @@ bool simple_wallet::save_watch_only(const std::vector<std::string> &args/* = std //---------------------------------------------------------------------------------------------------- bool simple_wallet::start_mining(const std::vector<std::string>& args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -3906,29 +4089,29 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args) if (args.size() == 2) { if (args[1] == "trusted") - m_trusted_daemon = true; + m_wallet->set_trusted_daemon(true); else if (args[1] == "untrusted") - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); else { fail_msg_writer() << tr("Expected trusted or untrusted, got ") << args[1] << ": assuming untrusted"; - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); } } else { - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); try { if (tools::is_local_address(m_wallet->get_daemon_address())) { MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; + m_wallet->set_trusted_daemon(true); } } catch (const std::exception &e) { } } - success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (*m_trusted_daemon ? tr("trusted") : tr("untrusted")); + success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (m_wallet->is_trusted_daemon() ? tr("trusted") : tr("untrusted")); } else { fail_msg_writer() << tr("This does not seem to be a valid daemon URL."); } @@ -3969,6 +4152,24 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, tr("txid ") << txid << ", " << print_money(amount) << ", " << tr("idx ") << subaddr_index; + + const uint64_t warn_height = m_wallet->nettype() == TESTNET ? 1000000 : m_wallet->nettype() == STAGENET ? 50000 : 1650000; + if (height >= warn_height) + { + std::vector<tx_extra_field> tx_extra_fields; + parse_tx_extra(tx.extra, tx_extra_fields); // failure ok + tx_extra_nonce extra_nonce; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + crypto::hash8 payment_id8 = crypto::null_hash8; + if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + message_writer() << + tr("NOTE: this transaction uses an encrypted payment ID: consider using subaddresses instead"); + else + message_writer(console_color_red, false) << + tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead"); + } + } if (m_auto_refresh_refreshing) m_cmd_binder.print_prompt(); else @@ -3997,6 +4198,32 @@ void simple_wallet::on_skip_transaction(uint64_t height, const crypto::hash &txi { } //---------------------------------------------------------------------------------------------------- +boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char *reason) +{ + // can't ask for password from a background thread + if (!m_in_manual_refresh.load(std::memory_order_relaxed)) + { + message_writer(console_color_red, false) << tr("Password needed - use the refresh command"); + m_cmd_binder.print_prompt(); + return boost::none; + } + +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + std::string msg = tr("Enter password"); + if (reason && *reason) + msg += std::string(" (") + reason + ")"; + auto pwd_container = tools::password_container::prompt(false, msg.c_str()); + if (!pwd_container) + { + MERROR("Failed to read password"); + return boost::none; + } + + return pwd_container->password(); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init) { if (!try_connect_to_daemon(is_init)) @@ -4020,7 +4247,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(is_daemon_trusted(), start_height, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -4311,7 +4538,7 @@ bool simple_wallet::show_blockchain_height(const std::vector<std::string>& args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_spent(const std::vector<std::string> &args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -4472,11 +4699,10 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_) { // "transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]" - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); std::vector<std::string> local_args = args_; @@ -4520,8 +4746,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } - const size_t min_args = (transfer_type == TransferLocked) ? 3 : 2; + const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); @@ -4530,38 +4761,38 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<uint8_t> extra; bool payment_id_seen = false; - bool expect_even = (transfer_type == TransferLocked); - if ((expect_even ? 0 : 1) == local_args.size() % 2) + if (!local_args.empty()) { std::string payment_id_str = local_args.back(); - local_args.pop_back(); - crypto::hash payment_id; - bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); - if(r) + bool r = true; + if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } else { crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) + if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) { std::string extra_nonce; set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; } } if(!r) { - fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + fail_msg_writer() << tr("payment id failed to encode"); return true; } - payment_id_seen = true; } uint64_t locked_blocks = 0; @@ -4586,11 +4817,54 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri vector<cryptonote::tx_destination_entry> dsts; size_t num_subaddresses = 0; - for (size_t i = 0; i < local_args.size(); i += 2) + for (size_t i = 0; i < local_args.size(); ) { - cryptonote::address_parse_info info; cryptonote::tx_destination_entry de; - if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter)) + cryptonote::address_parse_info info; + bool r = true; + + // check for a URI + std::string address_uri, payment_id_uri, tx_description, recipient_name, error; + std::vector<std::string> unknown_parameters; + uint64_t amount = 0; + bool has_uri = m_wallet->parse_uri(local_args[i], address_uri, payment_id_uri, amount, tx_description, recipient_name, unknown_parameters, error); + if (has_uri) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), address_uri, oa_prompter); + if (payment_id_uri.size() == 16) + { + if (!tools::wallet2::parse_short_payment_id(payment_id_uri, info.payment_id)) + { + fail_msg_writer() << tr("failed to parse short payment ID from URI"); + return true; + } + info.has_payment_id = true; + } + de.amount = amount; + ++i; + } + else if (i + 1 < local_args.size()) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter); + bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); + if(!ok || 0 == de.amount) + { + fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << + ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); + return true; + } + i += 2; + } + else + { + if (boost::starts_with(local_args[i], "monero:")) + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back() << ": " << error; + else + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back(); + return true; + } + + if (!r) { fail_msg_writer() << tr("failed to parse address"); return true; @@ -4599,16 +4873,30 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri de.is_subaddress = info.is_subaddress; num_subaddresses += info.is_subaddress; - if (info.has_payment_id) + if (info.has_payment_id || !payment_id_uri.empty()) { if (payment_id_seen) { - fail_msg_writer() << tr("a single transaction cannot use more than one payment id: ") << local_args[i]; + fail_msg_writer() << tr("a single transaction cannot use more than one payment id"); return true; } + crypto::hash payment_id; std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + if (info.has_payment_id) + { + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + } + else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) + { + set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + } + else + { + fail_msg_writer() << tr("failed to parse payment id, though it was detected"); + return true; + } bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); if(!r) { @@ -4618,14 +4906,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; } - bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); - if(!ok || 0 == de.amount) - { - fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << - ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); - return true; - } - dsts.push_back(de); } @@ -4659,17 +4939,14 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } unlock_block = bc_height + locked_blocks; - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); - break; - case TransferNew: - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); break; default: - LOG_ERROR("Unknown transfer method, using original"); + LOG_ERROR("Unknown transfer method, using default"); /* FALLTHRU */ - case TransferOriginal: - ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); - break; + case TransferNew: + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); + break; } if (ptx_vector.empty()) @@ -4844,7 +5121,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -4870,18 +5147,23 @@ bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) return transfer_main(TransferLocked, args_); } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) +{ + return sweep_main(0, true, args_); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); + try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(is_daemon_trusted()); + auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(); if (ptx_vector.empty()) { @@ -4960,13 +5242,13 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) { try { - m_wallet->discard_unmixable_outputs(is_daemon_trusted()); + m_wallet->discard_unmixable_outputs(); } catch (...) {} } } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -4977,7 +5259,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &args_) +bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args_) { auto print_usage = [below]() { @@ -4990,7 +5272,6 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; @@ -5039,10 +5320,50 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } + + uint64_t unlock_block = 0; + if (locked) { + uint64_t locked_blocks = 0; + + if (local_args.size() < 2) { + fail_msg_writer() << tr("missing lockedblocks parameter"); + return true; + } + + try + { + locked_blocks = boost::lexical_cast<uint64_t>(local_args[1]); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("bad locked_blocks parameter"); + return true; + } + if (locked_blocks > 1000000) + { + fail_msg_writer() << tr("Locked blocks too high, max 1000000 (˜4 yrs)"); + return true; + } + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (!err.empty()) + { + fail_msg_writer() << tr("failed to get blockchain height: ") << err; + return true; + } + unlock_block = bc_height + locked_blocks; + + local_args.erase(local_args.begin() + 1); + } std::vector<uint8_t> extra; bool payment_id_seen = false; - if (2 >= local_args.size()) + if (local_args.size() >= 2) { std::string payment_id_str = local_args.back(); @@ -5119,12 +5440,12 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } } - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); if (ptx_vector.empty()) { @@ -5208,7 +5529,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5221,7 +5542,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_single(const std::vector<std::string> &args_) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); if (!try_connect_to_daemon()) return true; @@ -5242,12 +5563,28 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIX; } + else if (ring_size == 0) + { + fail_msg_writer() << tr("Ring size must not be 0"); + return true; + } else { fake_outs_count = ring_size - 1; local_args.erase(local_args.begin()); } } + uint64_t adjusted_fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); + if (adjusted_fake_outs_count > fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } std::vector<uint8_t> extra; bool payment_id_seen = false; @@ -5337,7 +5674,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra); if (ptx_vector.empty()) { @@ -5407,7 +5744,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5420,7 +5757,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_all(const std::vector<std::string> &args_) { - return sweep_main(0, args_); + return sweep_main(0, false, args_); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_below(const std::vector<std::string> &args_) @@ -5436,7 +5773,7 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_) fail_msg_writer() << tr("invalid amount threshold"); return true; } - return sweep_main(below, std::vector<std::string>(++args_.begin(), args_.end())); + return sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector<std::string> &args_) @@ -5647,7 +5984,8 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("usage: sign_transfer [export_raw]"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); const bool export_raw = args_.size() == 1; std::vector<tools::wallet2::pending_tx> ptx; @@ -5712,7 +6050,7 @@ bool simple_wallet::submit_transfer(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5736,7 +6074,6 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) fail_msg_writer() << tr("usage: get_tx_key <txid>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } crypto::hash txid; if (!epee::string_tools::hex_to_pod(local_args[0], txid)) @@ -5745,7 +6082,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) return true; } - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; @@ -5765,6 +6102,64 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) } } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) +{ + std::vector<std::string> local_args = args_; + + if(local_args.size() != 2) { + fail_msg_writer() << tr("usage: set_tx_key <txid> <tx_key>"); + return true; + } + + crypto::hash txid; + if (!epee::string_tools::hex_to_pod(local_args[0], txid)) + { + fail_msg_writer() << tr("failed to parse txid"); + return true; + } + + crypto::secret_key tx_key; + std::vector<crypto::secret_key> additional_tx_keys; + try + { + if (!epee::string_tools::hex_to_pod(local_args[1].substr(0, 64), tx_key)) + { + fail_msg_writer() << tr("failed to parse tx_key"); + return true; + } + while(true) + { + local_args[1] = local_args[1].substr(64); + if (local_args[1].empty()) + break; + additional_tx_keys.resize(additional_tx_keys.size() + 1); + if (!epee::string_tools::hex_to_pod(local_args[1].substr(0, 64), additional_tx_keys.back())) + { + fail_msg_writer() << tr("failed to parse tx_key"); + return true; + } + } + } + catch (const std::out_of_range &e) + { + fail_msg_writer() << tr("failed to parse tx_key"); + return true; + } + + LOCK_IDLE_SCOPE(); + + try + { + m_wallet->set_tx_key(txid, tx_key, additional_tx_keys); + success_msg_writer() << tr("Tx key successfully stored."); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to store tx key: ") << e.what(); + } + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) { if (m_wallet->key_on_device()) @@ -5792,7 +6187,7 @@ bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); try { @@ -6007,7 +6402,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); try { @@ -6102,9 +6497,7 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } - - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); try { @@ -6205,7 +6598,7 @@ static std::string get_human_readable_timespan(std::chrono::seconds seconds) if (ts < 3600 * 24 * 30.5) return std::to_string((uint64_t)(ts / (3600 * 24))) + tr(" days"); if (ts < 3600 * 24 * 365.25) - return std::to_string((uint64_t)(ts / (3600 * 24 * 365.25))) + tr(" months"); + return std::to_string((uint64_t)(ts / (3600 * 24 * 30.5))) + tr(" months"); return tr("a long time"); } //---------------------------------------------------------------------------------------------------- @@ -6217,12 +6610,13 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) bool pending = true; bool failed = true; bool pool = true; + bool coinbase = true; uint64_t min_height = 0; uint64_t max_height = (uint64_t)-1; boost::optional<uint32_t> subaddr_index; if(local_args.size() > 4) { - fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); + fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); return true; } @@ -6235,19 +6629,24 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) local_args.erase(local_args.begin()); } else if (local_args[0] == "out" || local_args[0] == "outgoing") { - in = pool = false; + in = pool = coinbase = false; local_args.erase(local_args.begin()); } else if (local_args[0] == "pending") { - in = out = failed = false; + in = out = failed = coinbase = false; local_args.erase(local_args.begin()); } else if (local_args[0] == "failed") { - in = out = pending = pool = false; + in = out = pending = pool = coinbase = false; local_args.erase(local_args.begin()); } else if (local_args[0] == "pool") { - in = out = pending = failed = false; + in = out = pending = failed = coinbase = false; + local_args.erase(local_args.begin()); + } + else if (local_args[0] == "coinbase") { + in = out = pending = failed = pool = false; + coinbase = true; local_args.erase(local_args.begin()); } else if (local_args[0] == "all" || local_args[0] == "both") { @@ -6288,20 +6687,23 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) local_args.erase(local_args.begin()); } - std::multimap<uint64_t, std::pair<bool,std::string>> output; + std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>> output; PAUSE_READLINE(); - if (in) { + if (in || coinbase) { std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; m_wallet->get_payments(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); for (std::list<std::pair<crypto::hash, tools::wallet2::payment_details>>::const_iterator i = payments.begin(); i != payments.end(); ++i) { const tools::wallet2::payment_details &pd = i->second; + if (!pd.m_coinbase && !in) + continue; std::string payment_id = string_tools::pod_to_hex(i->first); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%25.25s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); + const std::string type = pd.m_coinbase ? tr("block") : tr("in"); + output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_green, type, (boost::format("%25.25s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); } } @@ -6334,20 +6736,23 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%25.25s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); + output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_magenta, tr("out"), (boost::format("%25.25s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); } } // print in and out sorted by height - for (std::map<uint64_t, std::pair<bool, std::string>>::const_iterator i = output.begin(); i != output.end(); ++i) { - message_writer(i->second.first ? console_color_green : console_color_magenta, false) << + for (std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>>::const_iterator i = output.begin(); i != output.end(); ++i) { + message_writer(std::get<0>(i->second), false) << boost::format("%8.8llu %6.6s %s") % - ((unsigned long long)i->first) % (i->second.first ? tr("in") : tr("out")) % i->second.second; + ((unsigned long long)i->first) % std::get<1>(i->second) % std::get<2>(i->second); } if (pool) { try { + m_in_manual_refresh.store(true, std::memory_order_relaxed); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); + m_wallet->update_pool_state(); std::list<std::pair<crypto::hash, tools::wallet2::pool_payment_details>> payments; m_wallet->get_unconfirmed_payments(payments, m_current_subaddress_account, subaddr_indices); @@ -6401,7 +6806,7 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) auto local_args = args_; std::set<uint32_t> subaddr_indices; - if (local_args.size() > 0 && local_args[0].substr(0, 6) != "index=") + if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") { if (!parse_subaddress_indices(local_args[0], subaddr_indices)) return true; @@ -6531,6 +6936,14 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { + message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); + message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); + std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } return refresh_main(0, true); } //---------------------------------------------------------------------------------------------------- @@ -6550,7 +6963,7 @@ void simple_wallet::wallet_idle_thread() { uint64_t fetched_blocks; if (try_connect_to_daemon(true)) - m_wallet->refresh(is_daemon_trusted(), 0, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks); } catch(...) {} m_auto_refresh_refreshing = false; @@ -7196,7 +7609,8 @@ bool simple_wallet::sign(const std::vector<std::string> &args) fail_msg_writer() << tr("This wallet is multisig and cannot sign"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -7265,14 +7679,14 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args) fail_msg_writer() << tr("wallet is watch-only and cannot export key images"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; try { - LOCK_IDLE_SCOPE(); if (!m_wallet->export_key_images(filename)) { fail_msg_writer() << tr("failed to save file ") << filename; @@ -7297,7 +7711,7 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -7332,6 +7746,31 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::hw_reconnect(const std::vector<std::string> &args) +{ + if (!m_wallet->key_on_device()) + { + fail_msg_writer() << tr("command only supported by HW wallet"); + return true; + } + + LOCK_IDLE_SCOPE(); + try + { + bool r = m_wallet->reconnect_device(); + if (!r){ + fail_msg_writer() << tr("Failed to reconnect device"); + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to reconnect device: ") << tr(e.what()); + return true; + } + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::export_outputs(const std::vector<std::string> &args) { if (m_wallet->key_on_device()) @@ -7344,12 +7783,12 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_outputs <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; - LOCK_IDLE_SCOPE(); try { std::string data = m_wallet->export_outputs_to_str(); @@ -7395,7 +7834,7 @@ bool simple_wallet::import_outputs(const std::vector<std::string> &args) try { - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); size_t n_outputs = m_wallet->import_outputs_from_str(data); success_msg_writer() << boost::lexical_cast<std::string>(n_outputs) << " outputs imported"; } @@ -7628,8 +8067,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_restore_multisig_wallet ); command_line::add_arg(desc_params, arg_non_deterministic ); command_line::add_arg(desc_params, arg_electrum_seed ); - command_line::add_arg(desc_params, arg_trusted_daemon); - command_line::add_arg(desc_params, arg_untrusted_daemon); command_line::add_arg(desc_params, arg_allow_mismatched_daemon_version); command_line::add_arg(desc_params, arg_restore_height); command_line::add_arg(desc_params, arg_do_not_relay); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 1f2765055..d50e4ce04 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -44,6 +44,7 @@ #include "cryptonote_basic/cryptonote_basic_impl.h" #include "wallet/wallet2.h" #include "console_handler.h" +#include "wipeable_string.h" #include "common/i18n.h" #include "common/password.h" #include "crypto/crypto.h" // for definition of crypto::secret_key @@ -91,13 +92,13 @@ namespace cryptonote //! \return Prompts user for password and verifies against local file. Logs on error and returns `none` boost::optional<tools::password_container> get_and_verify_password() const; - bool new_wallet(const boost::program_options::variables_map& vm, const crypto::secret_key& recovery_key, + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, const crypto::secret_key& recovery_key, bool recover, bool two_random, const std::string &old_language); - bool new_wallet(const boost::program_options::variables_map& vm, const cryptonote::account_public_address& address, + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, const cryptonote::account_public_address& address, const boost::optional<crypto::secret_key>& spendkey, const crypto::secret_key& viewkey); - bool new_wallet(const boost::program_options::variables_map& vm, - const std::string &multisig_keys, const std::string &old_language); - bool new_wallet(const boost::program_options::variables_map& vm, const std::string& device_name); + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, + const epee::wipeable_string &multisig_keys, const std::string &old_language); + boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm); bool open_wallet(const boost::program_options::variables_map& vm); bool close_wallet(); @@ -153,7 +154,8 @@ namespace cryptonote bool transfer(const std::vector<std::string> &args); bool transfer_new(const std::vector<std::string> &args); bool locked_transfer(const std::vector<std::string> &args); - bool sweep_main(uint64_t below, const std::vector<std::string> &args); + bool locked_sweep_all(const std::vector<std::string> &args); + bool sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args); bool sweep_all(const std::vector<std::string> &args); bool sweep_below(const std::vector<std::string> &args); bool sweep_single(const std::vector<std::string> &args); @@ -176,6 +178,7 @@ namespace cryptonote bool rescan_spent(const std::vector<std::string> &args); bool set_log(const std::vector<std::string> &args); bool get_tx_key(const std::vector<std::string> &args); + bool set_tx_key(const std::vector<std::string> &args); bool check_tx_key(const std::vector<std::string> &args); bool get_tx_proof(const std::vector<std::string> &args); bool check_tx_proof(const std::vector<std::string> &args); @@ -198,6 +201,7 @@ namespace cryptonote bool verify(const std::vector<std::string> &args); bool export_key_images(const std::vector<std::string> &args); bool import_key_images(const std::vector<std::string> &args); + bool hw_reconnect(const std::vector<std::string> &args); bool export_outputs(const std::vector<std::string> &args); bool import_outputs(const std::vector<std::string> &args); bool show_transfer(const std::vector<std::string> &args); @@ -230,13 +234,12 @@ namespace cryptonote bool print_ring_members(const std::vector<tools::wallet2::pending_tx>& ptx_vector, std::ostream& ostr); std::string get_prompt() const; bool print_seed(bool encrypted); - bool is_daemon_trusted() const { return *m_trusted_daemon; } /*! * \brief Prints the seed with a nice message * \param seed seed to print */ - void print_seed(std::string seed); + void print_seed(const epee::wipeable_string &seed); /*! * \brief Gets the word seed language from the user. @@ -259,6 +262,7 @@ namespace cryptonote virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index); virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); + virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); //---------------------------------------------------------- friend class refresh_progress_reporter_t; @@ -327,13 +331,12 @@ namespace cryptonote std::string m_import_path; std::string m_subaddress_lookahead; - std::string m_electrum_seed; // electrum-style seed parameter + epee::wipeable_string m_electrum_seed; // electrum-style seed parameter crypto::secret_key m_recovery_key; // recovery key (used as random for wallet gen) bool m_restore_deterministic_wallet; // recover flag bool m_restore_multisig_wallet; // recover flag bool m_non_deterministic; // old 2-random generation - boost::optional<bool> m_trusted_daemon; bool m_allow_mismatched_daemon_version; bool m_restoring; // are we restoring, by whatever method? uint64_t m_restore_height; // optional diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index a16f4fe19..be10b9f62 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -90,6 +90,8 @@ target_link_libraries(wallet_rpc_server cncrypto common version + daemonizer + ${EPEE_READLINE} ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/wallet/api/utils.cpp b/src/wallet/api/utils.cpp index aebe41e59..86fe56564 100644 --- a/src/wallet/api/utils.cpp +++ b/src/wallet/api/utils.cpp @@ -51,6 +51,9 @@ bool isAddressLocal(const std::string &address) void onStartup() { tools::on_startup(); +#ifdef NDEBUG + tools::disable_core_dumps(); +#endif } } diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 3f6bfec9e..463185284 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -366,10 +366,9 @@ void Wallet::error(const std::string &category, const std::string &str) { } ///////////////////////// WalletImpl implementation //////////////////////// -WalletImpl::WalletImpl(NetworkType nettype) +WalletImpl::WalletImpl(NetworkType nettype, uint64_t kdf_rounds) :m_wallet(nullptr) , m_status(Wallet::Status_Ok) - , m_trustedDaemon(false) , m_wallet2Callback(nullptr) , m_recoveringFromSeed(false) , m_recoveringFromDevice(false) @@ -377,7 +376,7 @@ WalletImpl::WalletImpl(NetworkType nettype) , m_rebuildWalletCache(false) , m_is_connected(false) { - m_wallet = new tools::wallet2(static_cast<cryptonote::network_type>(nettype)); + m_wallet = new tools::wallet2(static_cast<cryptonote::network_type>(nettype), kdf_rounds, true); m_history = new TransactionHistoryImpl(this); m_wallet2Callback = new Wallet2CallbackImpl(this); m_wallet->callback(m_wallet2Callback); @@ -630,7 +629,7 @@ bool WalletImpl::recoverFromDevice(const std::string &path, const std::string &p m_recoveringFromDevice = true; try { - m_wallet->restore(path, password, device_name); + m_wallet->restore(path, password, device_name, false); LOG_PRINT_L1("Generated new wallet from device: " + device_name); } catch (const std::exception& e) { @@ -733,10 +732,10 @@ bool WalletImpl::close(bool store) std::string WalletImpl::seed() const { - std::string seed; + epee::wipeable_string seed; if (m_wallet) m_wallet->get_seed(seed); - return seed; + return std::string(seed.data(), seed.size()); // TODO } std::string WalletImpl::getSeedLanguage() const @@ -1245,6 +1244,20 @@ size_t WalletImpl::importMultisigImages(const vector<string>& images) { return 0; } +bool WalletImpl::hasMultisigPartialKeyImages() const { + try { + clearStatus(); + checkMultisigWalletReady(m_wallet); + + return m_wallet->has_multisig_partial_key_images(); + } catch (const exception& e) { + LOG_ERROR("Error on checking for partial multisig key images: ") << e.what(); + setStatusError(string(tr("Failed to check for partial multisig key images: ")) + e.what()); + } + + return false; +} + PendingTransaction* WalletImpl::restoreMultisigTransaction(const string& signData) { try { clearStatus(); @@ -1298,6 +1311,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIXIN; + fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); uint32_t adjusted_priority = m_wallet->adjust_priority(static_cast<uint32_t>(priority)); @@ -1358,7 +1372,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const dsts.push_back(de); transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } else { // for the GUI, sweep_all (i.e. amount set as "(all)") will always sweep all the funds in all the addresses if (subaddr_indices.empty()) @@ -1368,7 +1382,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const } transaction->m_pending_tx = m_wallet->create_transactions_all(0, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } if (multisig().isMultisig) { @@ -1381,8 +1395,8 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error &e) { - setStatusError((boost::format(tr("failed to get random outputs to mix: %s")) % e.what()).str()); + } catch (const tools::error::get_outs_error &e) { + setStatusError((boost::format(tr("failed to get outputs to mix: %s")) % e.what()).str()); } catch (const tools::error::not_enough_unlocked_money& e) { std::ostringstream writer; @@ -1454,7 +1468,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() do { try { - transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(m_trustedDaemon); + transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -1463,8 +1477,8 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error&) { - setStatusError(tr("failed to get random outputs to mix")); + } catch (const tools::error::get_outs_error&) { + setStatusError(tr("failed to get outputs to mix")); } catch (const tools::error::not_enough_unlocked_money& e) { setStatusError(""); std::ostringstream writer; @@ -1891,12 +1905,12 @@ Wallet::ConnectionStatus WalletImpl::connected() const void WalletImpl::setTrustedDaemon(bool arg) { - m_trustedDaemon = arg; + m_wallet->set_trusted_daemon(arg); } bool WalletImpl::trustedDaemon() const { - return m_trustedDaemon; + return m_wallet->is_trusted_daemon(); } bool WalletImpl::watchOnly() const @@ -2032,6 +2046,7 @@ bool WalletImpl::isNewWallet() const bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) { + // claim RPC so there's no in-memory encryption for now if (!m_wallet->init(daemon_address, m_daemon_login, upper_transaction_size_limit, ssl)) return false; @@ -2094,21 +2109,36 @@ bool WalletImpl::useForkRules(uint8_t version, int64_t early_blocks) const return m_wallet->use_fork_rules(version,early_blocks); } -bool WalletImpl::blackballOutputs(const std::vector<std::string> &pubkeys, bool add) +bool WalletImpl::blackballOutputs(const std::vector<std::string> &outputs, bool add) { - std::vector<crypto::public_key> raw_pubkeys; - raw_pubkeys.reserve(pubkeys.size()); - for (const std::string &str: pubkeys) + std::vector<std::pair<uint64_t, uint64_t>> raw_outputs; + raw_outputs.reserve(outputs.size()); + uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets; + for (const std::string &str: outputs) { - crypto::public_key pkey; - if (!epee::string_tools::hex_to_pod(str, pkey)) + if (sscanf(str.c_str(), "@%" PRIu64, &amount) == 1) + continue; + if (amount == std::numeric_limits<uint64_t>::max()) { - setStatusError(tr("Failed to parse output public key")); - return false; + setStatusError("First line is not an amount"); + return true; + } + if (sscanf(str.c_str(), "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets--) + raw_outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(str.c_str(), "%" PRIu64, &offset) == 1) + { + raw_outputs.push_back(std::make_pair(amount, offset)); + } + else + { + setStatusError(tr("Invalid output: ") + str); + return false; } - raw_pubkeys.push_back(pkey); } - bool ret = m_wallet->set_blackballed_outputs(raw_pubkeys, add); + bool ret = m_wallet->set_blackballed_outputs(raw_outputs, add); if (!ret) { setStatusError(tr("Failed to set blackballed outputs")); @@ -2117,15 +2147,20 @@ bool WalletImpl::blackballOutputs(const std::vector<std::string> &pubkeys, bool return true; } -bool WalletImpl::unblackballOutput(const std::string &pubkey) +bool WalletImpl::unblackballOutput(const std::string &amount, const std::string &offset) { - crypto::public_key raw_pubkey; - if (!epee::string_tools::hex_to_pod(pubkey, raw_pubkey)) + uint64_t raw_amount, raw_offset; + if (!epee::string_tools::get_xtype_from_string(raw_amount, amount)) { - setStatusError(tr("Failed to parse output public key")); + setStatusError(tr("Failed to parse output amount")); return false; } - bool ret = m_wallet->unblackball_output(raw_pubkey); + if (!epee::string_tools::get_xtype_from_string(raw_offset, offset)) + { + setStatusError(tr("Failed to parse output offset")); + return false; + } + bool ret = m_wallet->unblackball_output(std::make_pair(raw_amount, raw_offset)); if (!ret) { setStatusError(tr("Failed to unblackball output")); @@ -2205,6 +2240,20 @@ void WalletImpl::keyReuseMitigation2(bool mitigation) m_wallet->key_reuse_mitigation2(mitigation); } +bool WalletImpl::lockKeysFile() +{ + return m_wallet->lock_keys_file(); +} + +bool WalletImpl::unlockKeysFile() +{ + return m_wallet->unlock_keys_file(); +} + +bool WalletImpl::isKeysFileLocked() +{ + return m_wallet->is_keys_file_locked(); +} } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index eefb2fe94..2cbfa30d0 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -52,7 +52,7 @@ struct Wallet2CallbackImpl; class WalletImpl : public Wallet { public: - WalletImpl(NetworkType nettype = MAINNET); + WalletImpl(NetworkType nettype = MAINNET, uint64_t kdf_rounds = 1); ~WalletImpl(); bool create(const std::string &path, const std::string &password, const std::string &language); @@ -139,6 +139,7 @@ public: bool finalizeMultisig(const std::vector<std::string>& extraMultisigInfo) override; bool exportMultisigImages(std::string& images) override; size_t importMultisigImages(const std::vector<std::string>& images) override; + bool hasMultisigPartialKeyImages() const override; PendingTransaction* restoreMultisigTransaction(const std::string& signData) override; PendingTransaction * createTransaction(const std::string &dst_addr, const std::string &payment_id, @@ -181,13 +182,16 @@ public: virtual bool lightWalletLogin(bool &isNewWallet) const override; virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status) override; virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add) override; - virtual bool unblackballOutput(const std::string &pubkey) override; + virtual bool unblackballOutput(const std::string &amount, const std::string &offset) override; virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const override; virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const override; virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) override; virtual void segregatePreForkOutputs(bool segregate) override; virtual void segregationHeight(uint64_t height) override; virtual void keyReuseMitigation2(bool mitigation) override; + virtual bool lockKeysFile() override; + virtual bool unlockKeysFile() override; + virtual bool isKeysFileLocked() override; private: void clearStatus() const; @@ -216,7 +220,6 @@ private: mutable std::string m_errorString; std::string m_password; TransactionHistoryImpl * m_history; - bool m_trustedDaemon; Wallet2CallbackImpl * m_wallet2Callback; AddressBookImpl * m_addressBook; SubaddressImpl * m_subaddress; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index f54255e91..933916076 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -720,6 +720,11 @@ struct Wallet * @return number of imported images */ virtual size_t importMultisigImages(const std::vector<std::string>& images) = 0; + /** + * @brief hasMultisigPartialKeyImages - checks if wallet needs to import multisig key images from other participants + * @return true if there are partial key images + */ + virtual bool hasMultisigPartialKeyImages() const = 0; /** * @brief restoreMultisigTransaction creates PendingTransaction from signData @@ -875,7 +880,7 @@ struct Wallet virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add) = 0; //! unblackballs an output - virtual bool unblackballOutput(const std::string &pubkey) = 0; + virtual bool unblackballOutput(const std::string &amount, const std::string &offset) = 0; //! gets the ring used for a key image, if any virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const = 0; @@ -900,6 +905,12 @@ struct Wallet //! Initiates a light wallet import wallet request virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status) = 0; + + //! locks/unlocks the keys file; returns true on success + virtual bool lockKeysFile() = 0; + virtual bool unlockKeysFile() = 0; + //! returns true if the keys file is locked + virtual bool isKeysFileLocked() = 0; }; /** @@ -914,9 +925,10 @@ struct WalletManager * \param password Password of wallet file * \param language Language to be used to generate electrum seed mnemonic * \param nettype Network type + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if created successfully) */ - virtual Wallet * createWallet(const std::string &path, const std::string &password, const std::string &language, NetworkType nettype) = 0; + virtual Wallet * createWallet(const std::string &path, const std::string &password, const std::string &language, NetworkType nettype, uint64_t kdf_rounds = 1) = 0; Wallet * createWallet(const std::string &path, const std::string &password, const std::string &language, bool testnet = false) // deprecated { return createWallet(path, password, language, testnet ? TESTNET : MAINNET); @@ -927,9 +939,10 @@ struct WalletManager * \param path Name of wallet file * \param password Password of wallet file * \param nettype Network type + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if opened successfully) */ - virtual Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype) = 0; + virtual Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype, uint64_t kdf_rounds = 1) = 0; Wallet * openWallet(const std::string &path, const std::string &password, bool testnet = false) // deprecated { return openWallet(path, password, testnet ? TESTNET : MAINNET); @@ -942,10 +955,11 @@ struct WalletManager * \param mnemonic mnemonic (25 words electrum seed) * \param nettype Network type * \param restoreHeight restore from start height + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) */ virtual Wallet * recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, - NetworkType nettype = MAINNET, uint64_t restoreHeight = 0) = 0; + NetworkType nettype = MAINNET, uint64_t restoreHeight = 0, uint64_t kdf_rounds = 1) = 0; Wallet * recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, bool testnet = false, uint64_t restoreHeight = 0) // deprecated { @@ -977,6 +991,7 @@ struct WalletManager * \param addressString public address * \param viewKeyString view key * \param spendKeyString spend key (optional) + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) */ virtual Wallet * createWalletFromKeys(const std::string &path, @@ -986,7 +1001,8 @@ struct WalletManager uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString = "") = 0; + const std::string &spendKeyString = "", + uint64_t kdf_rounds = 1) = 0; Wallet * createWalletFromKeys(const std::string &path, const std::string &password, const std::string &language, @@ -1037,6 +1053,7 @@ struct WalletManager * \param deviceName Device name * \param restoreHeight restore from start height (0 sets to current height) * \param subaddressLookahead Size of subaddress lookahead (empty sets to some default low value) + * \param kdf_rounds Number of rounds for key derivation function * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) */ virtual Wallet * createWalletFromDevice(const std::string &path, @@ -1044,7 +1061,8 @@ struct WalletManager NetworkType nettype, const std::string &deviceName, uint64_t restoreHeight = 0, - const std::string &subaddressLookahead = "") = 0; + const std::string &subaddressLookahead = "", + uint64_t kdf_rounds = 1) = 0; /*! * \brief Closes wallet. In case operation succeeded, wallet object deleted. in case operation failed, wallet object not deleted @@ -1069,9 +1087,14 @@ struct WalletManager * @param keys_file_name - location of keys file * @param password - password to verify * @param no_spend_key - verify only view keys? + * @param kdf_rounds - number of rounds for key derivation function * @return - true if password is correct + * + * @note + * This function will fail when the wallet keys file is opened because the wallet program locks the keys file. + * In this case, Wallet::unlockKeysFile() and Wallet::lockKeysFile() need to be called before and after the call to this function, respectively. */ - virtual bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const = 0; + virtual bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key, uint64_t kdf_rounds = 1) const = 0; /*! * \brief findWallets - searches for the wallet files by given path name recursively diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 99eadc82f..3851ca9cc 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -50,16 +50,16 @@ namespace epee { namespace Monero { Wallet *WalletManagerImpl::createWallet(const std::string &path, const std::string &password, - const std::string &language, NetworkType nettype) + const std::string &language, NetworkType nettype, uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); wallet->create(path, password, language); return wallet; } -Wallet *WalletManagerImpl::openWallet(const std::string &path, const std::string &password, NetworkType nettype) +Wallet *WalletManagerImpl::openWallet(const std::string &path, const std::string &password, NetworkType nettype, uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); wallet->open(path, password); //Refresh addressBook wallet->addressBook()->refresh(); @@ -87,9 +87,10 @@ Wallet *WalletManagerImpl::recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, NetworkType nettype, - uint64_t restoreHeight) + uint64_t restoreHeight, + uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); } @@ -104,9 +105,10 @@ Wallet *WalletManagerImpl::createWalletFromKeys(const std::string &path, uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString) + const std::string &spendKeyString, + uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); } @@ -119,9 +121,10 @@ Wallet *WalletManagerImpl::createWalletFromDevice(const std::string &path, NetworkType nettype, const std::string &deviceName, uint64_t restoreHeight, - const std::string &subaddressLookahead) + const std::string &subaddressLookahead, + uint64_t kdf_rounds) { - WalletImpl * wallet = new WalletImpl(nettype); + WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); } @@ -159,9 +162,9 @@ bool WalletManagerImpl::walletExists(const std::string &path) return false; } -bool WalletManagerImpl::verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const +bool WalletManagerImpl::verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key, uint64_t kdf_rounds) const { - return tools::wallet2::verify_password(keys_file_name, password, no_spend_key, hw::get_device("default")); + return tools::wallet2::verify_password(keys_file_name, password, no_spend_key, hw::get_device("default"), kdf_rounds); } std::vector<std::string> WalletManagerImpl::findWallets(const std::string &path) diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index 656a7142c..8b1c8be7f 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -39,13 +39,14 @@ class WalletManagerImpl : public WalletManager { public: Wallet * createWallet(const std::string &path, const std::string &password, - const std::string &language, NetworkType nettype) override; - Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype) override; + const std::string &language, NetworkType nettype, uint64_t kdf_rounds = 1) override; + Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype, uint64_t kdf_rounds = 1) override; virtual Wallet * recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, NetworkType nettype, - uint64_t restoreHeight) override; + uint64_t restoreHeight, + uint64_t kdf_rounds = 1) override; virtual Wallet * createWalletFromKeys(const std::string &path, const std::string &password, const std::string &language, @@ -53,7 +54,8 @@ public: uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString = "") override; + const std::string &spendKeyString = "", + uint64_t kdf_rounds = 1) override; // next two methods are deprecated - use the above version which allow setting of a password virtual Wallet * recoveryWallet(const std::string &path, const std::string &mnemonic, NetworkType nettype, uint64_t restoreHeight) override; // deprecated: use createWalletFromKeys(..., password, ...) instead @@ -69,10 +71,11 @@ public: NetworkType nettype, const std::string &deviceName, uint64_t restoreHeight = 0, - const std::string &subaddressLookahead = "") override; + const std::string &subaddressLookahead = "", + uint64_t kdf_rounds = 1) override; virtual bool closeWallet(Wallet *wallet, bool store = true) override; bool walletExists(const std::string &path) override; - bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const override; + bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key, uint64_t kdf_rounds = 1) const override; std::vector<std::string> findWallets(const std::string &path) override; std::string errorString() const override; void setDaemonAddress(const std::string &address) override; diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 401ada61b..21f75371b 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -50,12 +50,13 @@ void NodeRPCProxy::invalidate() m_height = 0; for (size_t n = 0; n < 256; ++n) m_earliest_height[n] = 0; - m_dynamic_per_kb_fee_estimate = 0; - m_dynamic_per_kb_fee_estimate_cached_height = 0; - m_dynamic_per_kb_fee_estimate_grace_blocks = 0; + m_dynamic_base_fee_estimate = 0; + m_dynamic_base_fee_estimate_cached_height = 0; + m_dynamic_base_fee_estimate_grace_blocks = 0; + m_fee_quantization_mask = 1; m_rpc_version = 0; m_target_height = 0; - m_block_size_limit = 0; + m_block_weight_limit = 0; m_get_info_time = 0; } @@ -99,7 +100,7 @@ boost::optional<std::string> NodeRPCProxy::get_info() const CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get target blockchain height"); m_height = resp_t.height; m_target_height = resp_t.target_height; - m_block_size_limit = resp_t.block_size_limit; + m_block_weight_limit = resp_t.block_weight_limit ? resp_t.block_weight_limit : resp_t.block_size_limit; m_get_info_time = now; } return boost::optional<std::string>(); @@ -123,12 +124,12 @@ boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) c return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_block_size_limit(uint64_t &block_size_limit) const +boost::optional<std::string> NodeRPCProxy::get_block_weight_limit(uint64_t &block_weight_limit) const { auto res = get_info(); if (res) return res; - block_size_limit = m_block_size_limit; + block_weight_limit = m_block_weight_limit; return boost::optional<std::string>(); } @@ -146,14 +147,14 @@ boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version, CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get hard fork status"); - m_earliest_height[version] = resp_t.enabled ? resp_t.earliest_height : std::numeric_limits<uint64_t>::max(); + m_earliest_height[version] = resp_t.earliest_height; } earliest_height = m_earliest_height[version]; return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const +boost::optional<std::string> NodeRPCProxy::get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const { uint64_t height; @@ -161,10 +162,10 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 if (result) return result; - if (m_dynamic_per_kb_fee_estimate_cached_height != height || m_dynamic_per_kb_fee_estimate_grace_blocks != grace_blocks) + if (m_dynamic_base_fee_estimate_cached_height != height || m_dynamic_base_fee_estimate_grace_blocks != grace_blocks) { - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); req_t.grace_blocks = grace_blocks; @@ -173,12 +174,47 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); - m_dynamic_per_kb_fee_estimate = resp_t.fee; - m_dynamic_per_kb_fee_estimate_cached_height = height; - m_dynamic_per_kb_fee_estimate_grace_blocks = grace_blocks; + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_dynamic_base_fee_estimate_grace_blocks = grace_blocks; + m_fee_quantization_mask = resp_t.quantization_mask; } - fee = m_dynamic_per_kb_fee_estimate; + fee = m_dynamic_base_fee_estimate; + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_fee_quantization_mask(uint64_t &fee_quantization_mask) const +{ + uint64_t height; + + boost::optional<std::string> result = get_height(height); + if (result) + return result; + + if (m_dynamic_base_fee_estimate_cached_height != height) + { + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + + m_daemon_rpc_mutex.lock(); + req_t.grace_blocks = m_dynamic_base_fee_estimate_grace_blocks; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_fee_estimate", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_fee_quantization_mask = resp_t.quantization_mask; + } + + fee_quantization_mask = m_fee_quantization_mask; + if (fee_quantization_mask == 0) + { + MERROR("Fee quantization mask is 0, forcing to 1"); + fee_quantization_mask = 1; + } return boost::optional<std::string>(); } diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 8a65884f7..65f13eaaa 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -47,9 +47,10 @@ public: boost::optional<std::string> get_height(uint64_t &height) const; void set_height(uint64_t h); boost::optional<std::string> get_target_height(uint64_t &height) const; - boost::optional<std::string> get_block_size_limit(uint64_t &block_size_limit) const; + boost::optional<std::string> get_block_weight_limit(uint64_t &block_weight_limit) const; boost::optional<std::string> get_earliest_height(uint8_t version, uint64_t &earliest_height) const; - boost::optional<std::string> get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask) const; private: boost::optional<std::string> get_info() const; @@ -59,12 +60,13 @@ private: mutable uint64_t m_height; mutable uint64_t m_earliest_height[256]; - mutable uint64_t m_dynamic_per_kb_fee_estimate; - mutable uint64_t m_dynamic_per_kb_fee_estimate_cached_height; - mutable uint64_t m_dynamic_per_kb_fee_estimate_grace_blocks; + mutable uint64_t m_dynamic_base_fee_estimate; + mutable uint64_t m_dynamic_base_fee_estimate_cached_height; + mutable uint64_t m_dynamic_base_fee_estimate_grace_blocks; + mutable uint64_t m_fee_quantization_mask; mutable uint32_t m_rpc_version; mutable uint64_t m_target_height; - mutable uint64_t m_block_size_limit; + mutable uint64_t m_block_weight_limit; mutable time_t m_get_info_time; }; diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 3f2634c8b..e9fc6866d 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -55,6 +55,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } +static int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + return va < vb ? -1 : va > vb; +} + static std::string compress_ring(const std::vector<uint64_t> &ring) { std::string s; @@ -146,7 +153,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed) MDB_stat mst; int ret; - needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB ret = mdb_env_info(env, &mei); if (ret) @@ -217,9 +224,9 @@ ringdb::ringdb(std::string filename, const std::string &genesis): THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_compare(txn, dbi_rings, compare_hash32); - dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); + dbr = mdb_dbi_open(txn, ("blackballs2-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - mdb_set_dupsort(txn, dbi_blackballs, compare_hash32); + mdb_set_dupsort(txn, dbi_blackballs, compare_uint64); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); @@ -374,7 +381,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::blackball_worker(const crypto::public_key &output, int op) +bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op) { MDB_txn *txn; MDB_cursor *cursor; @@ -382,49 +389,61 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) bool tx_active = false; bool ret = true; - dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output"); + + dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; - MDB_val key = zerokeyval; - MDB_val data; - data.mv_data = (void*)&output; - data.mv_size = sizeof(output); - switch (op) + MDB_val key, data; + for (const std::pair<uint64_t, uint64_t> &output: outputs) + { + key.mv_data = (void*)&output.first; + key.mv_size = sizeof(output.first); + data.mv_data = (void*)&output.second; + data.mv_size = sizeof(output.second); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Blackballing output " << output.first << "/" << output.second); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_APPENDDUP); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Unblackballing output " << output.first << "/" << output.second); + dbr = mdb_del(txn, dbi_blackballs, &key, &data); + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + mdb_cursor_close(cursor); + break; + case BLACKBALL_CLEAR: + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + } + + if (op == BLACKBALL_CLEAR) { - case BLACKBALL_BLACKBALL: - MDEBUG("Blackballing output " << output); - dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); - if (dbr == MDB_KEYEXIST) - dbr = 0; - break; - case BLACKBALL_UNBLACKBALL: - MDEBUG("Unblackballing output " << output); - dbr = mdb_del(txn, dbi_blackballs, &key, &data); - if (dbr == MDB_NOTFOUND) - dbr = 0; - break; - case BLACKBALL_QUERY: - dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); - dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); - THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); - ret = dbr != MDB_NOTFOUND; - if (dbr == MDB_NOTFOUND) - dbr = 0; - mdb_cursor_close(cursor); - break; - case BLACKBALL_CLEAR: - dbr = mdb_drop(txn, dbi_blackballs, 0); - break; - default: - THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + dbr = mdb_drop(txn, dbi_blackballs, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr))); } - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); @@ -432,24 +451,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) return ret; } -bool ringdb::blackball(const crypto::public_key &output) +bool ringdb::blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs) +{ + return blackball_worker(outputs, BLACKBALL_BLACKBALL); +} + +bool ringdb::blackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_BLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_BLACKBALL); } -bool ringdb::unblackball(const crypto::public_key &output) +bool ringdb::unblackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_UNBLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_UNBLACKBALL); } -bool ringdb::blackballed(const crypto::public_key &output) +bool ringdb::blackballed(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_QUERY); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_QUERY); } bool ringdb::clear_blackballs() { - return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); + return blackball_worker(std::vector<std::pair<uint64_t, uint64_t>>(), BLACKBALL_CLEAR); } } diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 6b4bce124..7b448b0d7 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -49,13 +49,14 @@ namespace tools bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); - bool blackball(const crypto::public_key &output); - bool unblackball(const crypto::public_key &output); - bool blackballed(const crypto::public_key &output); + bool blackball(const std::pair<uint64_t, uint64_t> &output); + bool blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs); + bool unblackball(const std::pair<uint64_t, uint64_t> &output); + bool blackballed(const std::pair<uint64_t, uint64_t> &output); bool clear_blackballs(); private: - bool blackball_worker(const crypto::public_key &output, int op); + bool blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op); private: std::string filename; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index c2c02dd67..6502463e2 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -84,11 +84,12 @@ using namespace cryptonote; // used to choose when to stop adding outputs to a tx #define APPROXIMATE_INPUT_BYTES 80 -// used to target a given block size (additional outputs may be added on top to build fee) -#define TX_SIZE_TARGET(bytes) (bytes*2/3) +// used to target a given block weight (additional outputs may be added on top to build fee) +#define TX_WEIGHT_TARGET(bytes) (bytes*2/3) // arbitrary, used to generate different hashes from the same input #define CHACHA8_KEY_TAIL 0x8c +#define CACHE_KEY_TAIL 0x8d #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\004" #define SIGNED_TX_PREFIX "Monero signed tx set\004" @@ -121,8 +122,6 @@ using namespace cryptonote; static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; -std::atomic<unsigned int> tools::wallet2::key_ref::refs(0); - namespace { std::string get_default_ringdb_path() @@ -141,13 +140,14 @@ namespace struct options { const command_line::arg_descriptor<std::string> daemon_address = {"daemon-address", tools::wallet2::tr("Use daemon instance at <host>:<port>"), ""}; const command_line::arg_descriptor<std::string> daemon_host = {"daemon-host", tools::wallet2::tr("Use daemon instance at host <arg> instead of localhost"), ""}; + const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; + const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; const command_line::arg_descriptor<std::string> daemon_login = {"daemon-login", tools::wallet2::tr("Specify username[:password] for daemon RPC client"), "", true}; const command_line::arg_descriptor<bool> testnet = {"testnet", tools::wallet2::tr("For testnet. Daemon must also be launched with --testnet flag"), false}; const command_line::arg_descriptor<bool> stagenet = {"stagenet", tools::wallet2::tr("For stagenet. Daemon must also be launched with --stagenet flag"), false}; - const command_line::arg_descriptor<bool> restricted = {"restricted-rpc", tools::wallet2::tr("Restricts to view-only commands"), false}; const command_line::arg_descriptor<std::string, false, true, 2> shared_ringdb_dir = { "shared-ringdb-dir", tools::wallet2::tr("Set shared ring database path"), get_default_ringdb_path(), @@ -160,6 +160,8 @@ struct options { return val; } }; + const command_line::arg_descriptor<uint64_t> kdf_rounds = {"kdf-rounds", tools::wallet2::tr("Number of rounds for the key derivation function"), 1}; + const command_line::arg_descriptor<std::string> hw_device = {"hw-device", tools::wallet2::tr("HW device to use"), ""}; }; void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file) @@ -182,31 +184,35 @@ uint64_t calculate_fee(uint64_t fee_per_kb, size_t bytes, uint64_t fee_multiplie return kB * fee_per_kb * fee_multiplier; } -uint64_t calculate_fee(uint64_t fee_per_kb, const cryptonote::blobdata &blob, uint64_t fee_multiplier) +uint64_t calculate_fee_from_weight(uint64_t base_fee, uint64_t weight, uint64_t fee_multiplier, uint64_t fee_quantization_mask) { - return calculate_fee(fee_per_kb, blob.size(), fee_multiplier); + uint64_t fee = weight * base_fee * fee_multiplier; + fee = (fee + fee_quantization_mask - 1) / fee_quantization_mask * fee_quantization_mask; + return fee; } -std::string get_size_string(size_t sz) +std::string get_weight_string(size_t weight) { - return std::to_string(sz) + " bytes (" + std::to_string((sz + 1023) / 1024) + " kB)"; + return std::to_string(weight) + " weight"; } -std::string get_size_string(const cryptonote::blobdata &tx) +std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_size) { - return get_size_string(tx.size()); + return get_weight_string(get_transaction_weight(tx, blob_size)); } -std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; - const bool restricted = command_line::get_arg(vm, opts.restricted); + const uint64_t kdf_rounds = command_line::get_arg(vm, opts.kdf_rounds); + THROW_WALLET_EXCEPTION_IF(kdf_rounds == 0, tools::error::wallet_internal_error, "KDF rounds must not be 0"); auto daemon_address = command_line::get_arg(vm, opts.daemon_address); auto daemon_host = command_line::get_arg(vm, opts.daemon_host); auto daemon_port = command_line::get_arg(vm, opts.daemon_port); + auto device_name = command_line::get_arg(vm, opts.hw_device); THROW_WALLET_EXCEPTION_IF(!daemon_address.empty() && !daemon_host.empty() && 0 != daemon_port, tools::error::wallet_internal_error, tools::wallet2::tr("can't specify daemon host or port more than once")); @@ -236,10 +242,32 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); - std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, restricted)); - wallet->init(std::move(daemon_address), std::move(login)); + boost::optional<bool> trusted_daemon; + if (!command_line::is_arg_defaulted(vm, opts.trusted_daemon) || !command_line::is_arg_defaulted(vm, opts.untrusted_daemon)) + trusted_daemon = command_line::get_arg(vm, opts.trusted_daemon) && !command_line::get_arg(vm, opts.untrusted_daemon); + THROW_WALLET_EXCEPTION_IF(!command_line::is_arg_defaulted(vm, opts.trusted_daemon) && !command_line::is_arg_defaulted(vm, opts.untrusted_daemon), + tools::error::wallet_internal_error, tools::wallet2::tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted")); + + // set --trusted-daemon if local and not overridden + if (!trusted_daemon) + { + try + { + trusted_daemon = false; + if (tools::is_local_address(daemon_address)) + { + MINFO(tr("Daemon is local, assuming trusted")); + trusted_daemon = true; + } + } + catch (const std::exception &e) { } + } + + std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds, unattended)); + wallet->init(std::move(daemon_address), std::move(login), 0, false, *trusted_daemon); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); + wallet->device_name(device_name); return wallet; } @@ -272,7 +300,7 @@ boost::optional<tools::password_container> get_password(const boost::program_opt return password_prompter(verify ? tr("Enter a new password for the wallet") : tr("Wallet password"), verify); } -std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); @@ -410,7 +438,7 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, THROW_WALLET_EXCEPTION_IF(deprecated_wallet, tools::error::wallet_internal_error, tools::wallet2::tr("Cannot generate deprecated wallets from JSON")); - wallet.reset(make_basic(vm, opts, password_prompter).release()); + wallet.reset(make_basic(vm, unattended, opts, password_prompter).release()); wallet->set_refresh_from_block_height(field_scan_from_height); wallet->explicit_refresh_from_block_height(field_scan_from_height_found); @@ -553,7 +581,12 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // rangeSigs if (bulletproof) - size += ((2*6 + 4 + 5)*32 + 3) * n_outputs; + { + size_t log_padded_outputs = 0; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + size += (2 * (6 + log_padded_outputs) + 4 + 5) * 32 + 3; + } else size += (2*64*32+32+64*32) * n_outputs; @@ -572,23 +605,63 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // txnFee size += 4; - LOG_PRINT_L2("estimated rct tx size for " << n_inputs << " with ring size " << (mixin+1) << " and " << n_outputs << ": " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); + LOG_PRINT_L2("estimated " << (bulletproof ? "bulletproof" : "borromean") << " rct tx size for " << n_inputs << " inputs with ring size " << (mixin+1) << " and " << n_outputs << " outputs: " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); return size; } size_t estimate_tx_size(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) { if (use_rct) - return estimate_rct_tx_size(n_inputs, mixin, n_outputs + 1, extra_size, bulletproof); + return estimate_rct_tx_size(n_inputs, mixin, n_outputs, extra_size, bulletproof); else return n_inputs * (mixin+1) * APPROXIMATE_INPUT_BYTES + extra_size; } +uint64_t estimate_tx_weight(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) +{ + size_t size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + if (use_rct && bulletproof && n_outputs > 2) + { + const uint64_t bp_base = 368; + size_t log_padded_outputs = 2; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + uint64_t nlr = 2 * (6 + log_padded_outputs); + const uint64_t bp_size = 32 * (9 + nlr); + const uint64_t bp_clawback = (bp_base * (1<<log_padded_outputs) - bp_size) * 4 / 5; + MDEBUG("clawback on size " << size << ": " << bp_clawback); + size += bp_clawback; + } + return size; +} + uint8_t get_bulletproof_fork() { return 8; } +uint64_t estimate_fee(bool use_per_byte_fee, bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + { + const size_t estimated_tx_weight = estimate_tx_weight(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee_from_weight(base_fee, estimated_tx_weight, fee_multiplier, fee_quantization_mask); + } + else + { + const size_t estimated_tx_size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee(base_fee, estimated_tx_size, fee_multiplier); + } +} + +uint64_t calculate_fee(bool use_per_byte_fee, const cryptonote::transaction &tx, size_t blob_size, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + return calculate_fee_from_weight(base_fee, cryptonote::get_transaction_weight(tx, blob_size), fee_multiplier, fee_quantization_mask); + else + return calculate_fee(base_fee, blob_size, fee_multiplier); +} + crypto::hash8 get_short_payment_id(const tools::wallet2::pending_tx &ptx, hw::device &hwdev) { crypto::hash8 payment_id8 = null_hash8; @@ -647,11 +720,43 @@ const size_t MAX_SPLIT_ATTEMPTS = 30; constexpr const std::chrono::seconds wallet2::rpc_timeout; const char* wallet2::tr(const char* str) { return i18n_translate(str, "tools::wallet2"); } -wallet2::wallet2(network_type nettype, bool restricted): +wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, const boost::optional<tools::password_container> &password): + w(w), + locked(password != boost::none) +{ + if (!locked || w.is_unattended() || w.ask_password() != tools::wallet2::AskPasswordToDecrypt) + { + locked = false; + return; + } + const epee::wipeable_string pass = password->password(); + w.generate_chacha_key_from_password(pass, key); + w.decrypt_keys(key); +} + +wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, bool locked, const epee::wipeable_string &password): + w(w), + locked(locked) +{ + if (!locked) + return; + w.generate_chacha_key_from_password(password, key); + w.decrypt_keys(key); +} + +wallet_keys_unlocker::~wallet_keys_unlocker() +{ + if (!locked) + return; + w.encrypt_keys(key); +} + +wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), m_run(true), m_callback(0), + m_trusted_daemon(false), m_nettype(nettype), m_always_confirm_transfers(true), m_print_ring_members(false), @@ -665,7 +770,7 @@ wallet2::wallet2(network_type nettype, bool restricted): m_explicit_refresh_from_block_height(true), m_confirm_missing_payment_id(true), m_confirm_non_default_ring_size(true), - m_ask_password(true), + m_ask_password(AskPasswordToDecrypt), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), @@ -678,7 +783,7 @@ wallet2::wallet2(network_type nettype, bool restricted): m_segregation_height(0), m_ignore_fractional_outputs(true), m_is_initialized(false), - m_restricted(restricted), + m_kdf_rounds(kdf_rounds), is_old_file_format(false), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex), m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR), @@ -692,7 +797,9 @@ wallet2::wallet2(network_type nettype, bool restricted): m_key_on_device(false), m_ring_history_saved(false), m_ringdb(), - m_last_block_reward(0) + m_last_block_reward(0), + m_encrypt_keys_after_refresh(boost::none), + m_unattended(unattended) { } @@ -710,29 +817,37 @@ bool wallet2::has_stagenet_option(const boost::program_options::variables_map& v return command_line::get_arg(vm, options().stagenet); } +std::string wallet2::device_name_option(const boost::program_options::variables_map& vm) +{ + return command_line::get_arg(vm, options().hw_device); +} + void wallet2::init_options(boost::program_options::options_description& desc_params) { const options opts{}; command_line::add_arg(desc_params, opts.daemon_address); command_line::add_arg(desc_params, opts.daemon_host); + command_line::add_arg(desc_params, opts.trusted_daemon); + command_line::add_arg(desc_params, opts.untrusted_daemon); command_line::add_arg(desc_params, opts.password); command_line::add_arg(desc_params, opts.password_file); command_line::add_arg(desc_params, opts.daemon_port); command_line::add_arg(desc_params, opts.daemon_login); command_line::add_arg(desc_params, opts.testnet); command_line::add_arg(desc_params, opts.stagenet); - command_line::add_arg(desc_params, opts.restricted); command_line::add_arg(desc_params, opts.shared_ringdb_dir); + command_line::add_arg(desc_params, opts.kdf_rounds); + command_line::add_arg(desc_params, opts.hw_device); } -std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, bool unattended, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; - return generate_from_json(json_file, vm, opts, password_prompter); + return generate_from_json(json_file, vm, unattended, opts, password_prompter); } std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( - const boost::program_options::variables_map& vm, const std::string& wallet_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) + const boost::program_options::variables_map& vm, bool unattended, const std::string& wallet_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; auto pwd = get_password(vm, opts, password_prompter, false); @@ -740,7 +855,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( { return {nullptr, password_container{}}; } - auto wallet = make_basic(vm, opts, password_prompter); + auto wallet = make_basic(vm, unattended, opts, password_prompter); if (wallet) { wallet->load(wallet_file, pwd->password()); @@ -748,7 +863,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( return {std::move(wallet), std::move(*pwd)}; } -std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter) +std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter) { const options opts{}; auto pwd = get_password(vm, opts, password_prompter, true); @@ -756,25 +871,26 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const { return {nullptr, password_container{}}; } - return {make_basic(vm, opts, password_prompter), std::move(*pwd)}; + return {make_basic(vm, unattended, opts, password_prompter), std::move(*pwd)}; } -std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::variables_map& vm, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; - return make_basic(vm, opts, password_prompter); + return make_basic(vm, unattended, opts, password_prompter); } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_size_limit, bool ssl) +bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool ssl, bool trusted_daemon) { m_checkpoints.init_default_checkpoints(m_nettype); if(m_http_client.is_connected()) m_http_client.disconnect(); m_is_initialized = true; - m_upper_transaction_size_limit = upper_transaction_size_limit; + m_upper_transaction_weight_limit = upper_transaction_weight_limit; m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); + m_trusted_daemon = trusted_daemon; // When switching from light wallet to full wallet, we need to reset the height we got from lw node. return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl); } @@ -784,11 +900,10 @@ bool wallet2::is_deterministic() const crypto::secret_key second; keccak((uint8_t *)&get_account().get_keys().m_spend_secret_key, sizeof(crypto::secret_key), (uint8_t *)&second, sizeof(crypto::secret_key)); sc_reduce32((uint8_t *)&second); - bool keys_deterministic = memcmp(second.data,get_account().get_keys().m_view_secret_key.data, sizeof(crypto::secret_key)) == 0; - return keys_deterministic; + return memcmp(second.data,get_account().get_keys().m_view_secret_key.data, sizeof(crypto::secret_key)) == 0; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_seed(std::string& electrum_words, const epee::wipeable_string &passphrase) const +bool wallet2::get_seed(epee::wipeable_string& electrum_words, const epee::wipeable_string &passphrase) const { bool keys_deterministic = is_deterministic(); if (!keys_deterministic) @@ -814,7 +929,7 @@ bool wallet2::get_seed(std::string& electrum_words, const epee::wipeable_string return true; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string &passphrase, bool raw) const +bool wallet2::get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase, bool raw) const { bool ready; uint32_t threshold, total; @@ -837,7 +952,7 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & crypto::secret_key skey; crypto::public_key pkey; const account_keys &keys = get_account().get_keys(); - std::string data; + epee::wipeable_string data; data.append((const char*)&threshold, sizeof(uint32_t)); data.append((const char*)&total, sizeof(uint32_t)); skey = keys.m_spend_secret_key; @@ -863,7 +978,7 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & if (raw) { - seed = epee::string_tools::buff_to_hex_nodelimer(data); + seed = epee::to_hex::wipeable_string({(const unsigned char*)data.data(), data.size()}); } else { @@ -877,6 +992,27 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & return true; } //---------------------------------------------------------------------------------------------------- +bool wallet2::reconnect_device() +{ + bool r = true; + hw::device &hwdev = hw::get_device(m_device_name); + hwdev.set_name(m_device_name); + r = hwdev.init(); + if (!r){ + LOG_PRINT_L2("Could not init device"); + return false; + } + + r = hwdev.connect(); + if (!r){ + LOG_PRINT_L2("Could not connect to the device"); + return false; + } + + m_account.set_device(hwdev); + return true; +} +//---------------------------------------------------------------------------------------------------- /*! * \brief Gets the seed language */ @@ -899,6 +1035,14 @@ cryptonote::account_public_address wallet2::get_subaddress(const cryptonote::sub return hwdev.get_subaddress(m_account.get_keys(), index); } //---------------------------------------------------------------------------------------------------- +boost::optional<cryptonote::subaddress_index> wallet2::get_subaddress_index(const cryptonote::account_public_address& address) const +{ + auto index = m_subaddresses.find(address.m_spend_public_key); + if (index == m_subaddresses.end()) + return boost::none; + return index->second; +} +//---------------------------------------------------------------------------------------------------- crypto::public_key wallet2::get_subaddress_spend_public_key(const cryptonote::subaddress_index& index) const { hw::device &hwdev = m_account.get_device(); @@ -1077,10 +1221,9 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & switch (rv.type) { case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: return rct::decodeRctSimple(rv, rct::sk2rct(scalar1), i, mask, hwdev); case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: return rct::decodeRct(rv, rct::sk2rct(scalar1), i, mask, hwdev); default: LOG_ERROR("Unsupported rct type: " << rv.type); @@ -1094,9 +1237,25 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & } } //---------------------------------------------------------------------------------------------------- -void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) const +void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) { THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); + + // if keys are encrypted, ask for password + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only && !m_multisig_rescan_k) + { + static critical_section password_lock; + CRITICAL_REGION_LOCAL(password_lock); + if (!m_encrypt_keys_after_refresh) + { + boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password("output received"); + THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); + THROW_WALLET_EXCEPTION_IF(!verify_password(*pwd), error::password_needed, tr("Invalid password: password is needed to compute key image for incoming monero")); + decrypt_keys(*pwd); + m_encrypt_keys_after_refresh = *pwd; + } + } + if (m_multisig) { tx_scan_info.in_ephemeral.pub = boost::get<cryptonote::txout_to_key>(tx.vout[i].target).key; @@ -1557,6 +1716,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { // We got a payment ID to go with this tx LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8); + MINFO("Consider using subaddresses instead of encrypted payment IDs"); if (tx_pub_key != null_pkey) { if (!m_account.get_device().decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key)) @@ -1580,12 +1740,9 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); + MWARNING("Found unencrypted payment ID: these are bad for privacy, consider using subaddresses instead"); } } - else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) - { - LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); - } for (const auto& i : tx_money_got_in_outs) { @@ -1596,6 +1753,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote payment.m_block_height = height; payment.m_unlock_time = tx.unlock_time; payment.m_timestamp = ts; + payment.m_coinbase = miner_tx; payment.m_subaddr_index = i.first; if (pool) { emplace_or_replace(m_unconfirmed_payments, payment_id, pool_payment_details{payment, double_spend_seen}); @@ -1755,34 +1913,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); req.block_ids = short_chain_history; - uint32_t rpc_version; - boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); - // no error - if (!!result) - { - // empty string -> not connection - THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion"); - THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion"); - if (*result != CORE_RPC_STATUS_OK) - { - MDEBUG("Cannot determine daemon RPC version, not asking for pruned blocks"); - req.prune = false; // old daemon - } - } - else - { - if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 7)) - { - MDEBUG("Daemon is recent enough, asking for pruned blocks"); - req.prune = true; - } - else - { - MDEBUG("Daemon is too old, not asking for pruned blocks"); - req.prune = false; - } - } - + req.prune = true; req.start_height = start_height; req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; m_daemon_rpc_mutex.lock(); @@ -1968,9 +2099,11 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks { drop_from_short_history(short_chain_history, 3); + THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); + // prepend the last 3 blocks, should be enough to guard against a block or two's reorg std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) + for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) { short_chain_history.push_front(i->hash); ++i; @@ -2055,6 +2188,14 @@ void wallet2::update_pool_state(bool refreshed) { MDEBUG("update_pool_state start"); + auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { + if (m_encrypt_keys_after_refresh) + { + encrypt_keys(*m_encrypt_keys_after_refresh); + m_encrypt_keys_after_refresh = boost::none; + } + }); + // get the pool state cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request req; cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response res; @@ -2282,6 +2423,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); + m_blockchain.trim(checkpoint_height); short_chain_history.clear(); get_short_chain_history(short_chain_history); } @@ -2361,8 +2503,6 @@ bool wallet2::delete_address_book_row(std::size_t row_id) { //---------------------------------------------------------------------------------------------------- void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { - key_ref kref(*this); - if(m_light_wallet) { // MyMonero get_address_info needs to be called occasionally to trigger wallet sync. @@ -2430,6 +2570,14 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo // subsequent pulls in this refresh. start_height = 0; + auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { + if (m_encrypt_keys_after_refresh) + { + encrypt_keys(*m_encrypt_keys_after_refresh); + m_encrypt_keys_after_refresh = boost::none; + } + }); + bool first = true; while(m_run.load(std::memory_order_relaxed)) { @@ -2487,10 +2635,6 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo break; } - // switch to the new blocks from the daemon - blocks_start_height = next_blocks_start_height; - blocks = std::move(next_blocks); - parsed_blocks = std::move(next_parsed_blocks); first = false; // handle error from async fetching thread @@ -2498,6 +2642,17 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo { throw std::runtime_error("proxy exception in refresh thread"); } + + // switch to the new blocks from the daemon + blocks_start_height = next_blocks_start_height; + blocks = std::move(next_blocks); + parsed_blocks = std::move(next_parsed_blocks); + } + catch (const tools::error::password_needed&) + { + blocks_fetched += added_blocks; + waiter.wait(&tpool); + throw; } catch (const std::exception&) { @@ -2549,7 +2704,7 @@ bool wallet2::refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& rece return ok; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) +bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) { uint32_t rpc_version; boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); @@ -2583,6 +2738,7 @@ bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64 req.amounts.push_back(0); req.from_height = 0; req.cumulative = true; + req.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2683,7 +2839,7 @@ void wallet2::detach_blockchain(uint64_t height) bool wallet2::deinit() { m_is_initialized=false; - m_keys_file_locker.reset(); + unlock_keys_file(); return true; } //---------------------------------------------------------------------------------------------------- @@ -2720,8 +2876,20 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable std::string multisig_signers; cryptonote::account_base account = m_account; + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + { + account.encrypt_viewkey(key); + account.decrypt_keys(key); + } + if (watch_only) account.forget_spend_key(); + + account.encrypt_keys(key); + bool r = epee::serialization::store_t_to_binary(account, account_data); CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet keys"); wallet2::keys_file_data keys_file_data = boost::value_initialized<wallet2::keys_file_data>(); @@ -2790,7 +2958,7 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetInt(m_confirm_non_default_ring_size ? 1 :0); json.AddMember("confirm_non_default_ring_size", value2, json.GetAllocator()); - value2.SetInt(m_ask_password ? 1 :0); + value2.SetInt(m_ask_password); json.AddMember("ask_password", value2, json.GetAllocator()); value2.SetUint(m_min_output_count); @@ -2838,6 +3006,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetUint(m_subaddress_lookahead_minor); json.AddMember("subaddress_lookahead_minor", value2, json.GetAllocator()); + value2.SetUint(1); + json.AddMember("encrypted_secret_keys", value2, json.GetAllocator()); + + value.SetString(m_device_name.c_str(), m_device_name.size()); + json.AddMember("device_name", value, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -2845,24 +3019,52 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable account_data = buffer.GetString(); // Encrypt the entire JSON object. - crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key); + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string cipher; cipher.resize(account_data.size()); keys_file_data.iv = crypto::rand<crypto::chacha_iv>(); crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); keys_file_data.account_data = cipher; - m_keys_file_locker.reset(); + unlock_keys_file(); std::string buf; r = ::serialization::dump_binary(keys_file_data, buf); r = r && epee::file_io_utils::save_string_to_file(keys_file_name, buf); //and never touch wallet_keys_file again, only read CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << keys_file_name); - m_keys_file_locker.reset(new tools::file_locker(m_keys_file)); + lock_keys_file(); return true; } //---------------------------------------------------------------------------------------------------- +void wallet2::setup_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + + // re-encrypt, but keep viewkey unencrypted + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + { + m_account.encrypt_keys(key); + m_account.decrypt_viewkey(key); + } + + static_assert(HASH_SIZE == sizeof(crypto::chacha_key), "Mismatched sizes of hash and chacha key"); + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE+1>> cache_key_data; + memcpy(cache_key_data.data(), &key, HASH_SIZE); + cache_key_data[HASH_SIZE] = CACHE_KEY_TAIL; + cn_fast_hash(cache_key_data.data(), HASH_SIZE+1, (crypto::hash&)m_cache_key); + get_ringdb_key(); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password) +{ + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + decrypt_keys(original_password); + setup_keys(new_password); + rewrite(filename, new_password); + store(); +} +//---------------------------------------------------------------------------------------------------- /*! * \brief Load wallet information from wallet file. * \param keys_file_name Name of wallet file @@ -2873,6 +3075,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ rapidjson::Document json; wallet2::keys_file_data keys_file_data; std::string buf; + bool encrypted_secret_keys = false; bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); @@ -2880,7 +3083,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ r = ::serialization::parse_binary(buf, keys_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key); + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); @@ -2903,7 +3106,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_refresh_type = RefreshType::RefreshDefault; m_confirm_missing_payment_id = true; m_confirm_non_default_ring_size = true; - m_ask_password = true; + m_ask_password = AskPasswordToDecrypt; m_min_output_count = 0; m_min_output_value = 0; m_merge_destinations = false; @@ -2917,7 +3120,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_ignore_fractional_outputs = true; m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR; m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; + m_device_name = ""; m_key_on_device = false; + encrypted_secret_keys = false; } else if(json.IsObject()) { @@ -3011,7 +3216,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_confirm_missing_payment_id = field_confirm_missing_payment_id; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, confirm_non_default_ring_size, int, Int, false, true); m_confirm_non_default_ring_size = field_confirm_non_default_ring_size; - GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ask_password, int, Int, false, true); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ask_password, AskPasswordType, Int, false, AskPasswordToDecrypt); m_ask_password = field_ask_password; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, default_decimal_point, int, Int, false, CRYPTONOTE_DISPLAY_DECIMAL_POINT); cryptonote::set_default_decimal_point(field_default_decimal_point); @@ -3047,6 +3252,15 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_subaddress_lookahead_major = field_subaddress_lookahead_major; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_minor, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MINOR); m_subaddress_lookahead_minor = field_subaddress_lookahead_minor; + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, encrypted_secret_keys, uint32_t, Uint, false, false); + encrypted_secret_keys = field_encrypted_secret_keys; + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_name, std::string, String, false, std::string()); + if (m_device_name.empty() && field_device_name_found) + { + m_device_name = field_device_name; + } } else { @@ -3057,18 +3271,46 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ r = epee::serialization::load_t_from_binary(m_account, account_data); if (r && m_key_on_device) { LOG_PRINT_L0("Account on device. Initing device..."); - hw::device &hwdev = hw::get_device("Ledger"); + hw::device &hwdev = hw::get_device(m_device_name); + hwdev.set_name(m_device_name); hwdev.init(); hwdev.connect(); m_account.set_device(hwdev); LOG_PRINT_L0("Device inited..."); } + + if (r) + { + if (encrypted_secret_keys) + { + m_account.decrypt_keys(key); + } + else + { + // rewrite with encrypted keys, ignore errors + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + encrypt_keys(key); + bool saved_ret = store_keys(keys_file_name, password, m_watch_only); + if (!saved_ret) + { + // just moan a bit, but not fatal + MERROR("Error saving keys file with encrypted keys, not fatal"); + } + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + decrypt_keys(key); + m_keys_file_locker.reset(); + } + } const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); if(!m_watch_only && !m_multisig) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); + + if (r) + setup_keys(password); + return true; } @@ -3082,9 +3324,13 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const epee::wipeable_string& password) const +bool wallet2::verify_password(const epee::wipeable_string& password) { - return verify_password(m_keys_file, password, m_watch_only || m_multisig, m_account.get_device()); + // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). + unlock_keys_file(); + bool r = verify_password(m_keys_file, password, m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); + lock_keys_file(); + return r; } /*! @@ -3100,11 +3346,12 @@ bool wallet2::verify_password(const epee::wipeable_string& password) const * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev) +bool wallet2::verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds) { rapidjson::Document json; wallet2::keys_file_data keys_file_data; std::string buf; + bool encrypted_secret_keys = false; bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); @@ -3112,7 +3359,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip r = ::serialization::parse_binary(buf, keys_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); crypto::chacha_key key; - crypto::generate_chacha_key(password.data(), password.size(), key); + crypto::generate_chacha_key(password.data(), password.size(), key, kdf_rounds); std::string account_data; account_data.resize(keys_file_data.account_data.size()); crypto::chacha20(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); @@ -3128,19 +3375,75 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip { account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() + json["key_data"].GetStringLength()); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, encrypted_secret_keys, uint32_t, Uint, false, false); + encrypted_secret_keys = field_encrypted_secret_keys; } cryptonote::account_base account_data_check; r = epee::serialization::load_t_from_binary(account_data_check, account_data); - const cryptonote::account_keys& keys = account_data_check.get_keys(); + if (encrypted_secret_keys) + account_data_check.decrypt_keys(key); + + const cryptonote::account_keys& keys = account_data_check.get_keys(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); if(!no_spend_key) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); return r; } +void wallet2::encrypt_keys(const crypto::chacha_key &key) +{ + m_account.encrypt_keys(key); + m_account.decrypt_viewkey(key); +} + +void wallet2::decrypt_keys(const crypto::chacha_key &key) +{ + m_account.encrypt_viewkey(key); + m_account.decrypt_keys(key); +} + +void wallet2::encrypt_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + encrypt_keys(key); +} + +void wallet2::decrypt_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + decrypt_keys(key); +} + +void wallet2::setup_new_blockchain() +{ + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); + add_subaddress_account(tr("Primary account")); +} + +void wallet2::create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file) +{ + if (!wallet_.empty()) + { + bool r = store_keys(m_keys_file, password, watch_only); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + if (create_address_file) + { + r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); + if(!r) MERROR("String with address text not saved"); + } + } +} + + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -3149,7 +3452,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip * \param create_address_file Whether to create an address file */ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& password, - const std::string& multisig_data, bool create_address_file) + const epee::wipeable_string& multisig_data, bool create_address_file) { clear(); prepare_file_names(wallet_); @@ -3205,6 +3508,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& for (const auto &msk: multisig_keys) sc_add(skey.bytes, skey.bytes, rct::sk2rct(msk).bytes); THROW_WALLET_EXCEPTION_IF(!(rct::rct2sk(skey) == spend_secret_key), error::invalid_multisig_seed); + memwipe(&skey, sizeof(rct::key)); m_account.make_multisig(view_secret_key, spend_secret_key, spend_public_key, multisig_keys); m_account.finalize_multisig(spend_public_key); @@ -3215,24 +3519,10 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig_threshold = threshold; m_multisig_signers = multisig_signers; m_key_on_device = false; + setup_keys(password); - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); - add_subaddress_account(tr("Primary account")); + create_keys_file(wallet_, false, password, m_nettype != MAINNET || create_address_file); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3269,29 +3559,16 @@ crypto::secret_key wallet2::generate(const std::string& wallet_, const epee::wip m_multisig_threshold = 0; m_multisig_signers.clear(); m_key_on_device = false; + setup_keys(password); // calculate a starting refresh height if(m_refresh_from_block_height == 0 && !recover){ m_refresh_from_block_height = estimate_blockchain_height(); } - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + create_keys_file(wallet_, false, password, m_nettype != MAINNET || create_address_file); - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3370,24 +3647,11 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig_threshold = 0; m_multisig_signers.clear(); m_key_on_device = false; + setup_keys(password); - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, true); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + create_keys_file(wallet_, true, password, m_nettype != MAINNET || create_address_file); - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3423,24 +3687,11 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig_threshold = 0; m_multisig_signers.clear(); m_key_on_device = false; + setup_keys(password); - if (!wallet_.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - - if (m_nettype != MAINNET || create_address_file) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } + create_keys_file(wallet_, false, password, create_address_file); - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) store(); @@ -3452,7 +3703,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& * \param password Password of wallet file * \param device_name device string address */ -void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name) +void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name, bool create_address_file) { clear(); prepare_file_names(wallet_); @@ -3463,31 +3714,27 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(m_keys_file, ignored_ec), error::file_exists, m_keys_file); } m_key_on_device = true; - m_account.create_from_device(device_name); + + auto &hwdev = hw::get_device(device_name); + hwdev.set_name(device_name); + + m_account.create_from_device(hwdev); m_account_public_address = m_account.get_keys().m_account_address; m_watch_only = false; m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + setup_keys(password); + m_device_name = device_name; - if (!wallet_.empty()) { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); + create_keys_file(wallet_, false, password, m_nettype != MAINNET || create_address_file); if (m_subaddress_lookahead_major == SUBADDRESS_LOOKAHEAD_MAJOR && m_subaddress_lookahead_minor == SUBADDRESS_LOOKAHEAD_MINOR) { // the default lookahead setting (50:200) is clearly too much for hardware wallet m_subaddress_lookahead_major = 5; m_subaddress_lookahead_minor = 20; } - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!wallet_.empty()) { store(); } @@ -3508,6 +3755,17 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, clear(); + // decrypt keys + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + { + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(password.data(), password.size(), chacha_key, m_kdf_rounds); + m_account.encrypt_viewkey(chacha_key); + m_account.decrypt_keys(chacha_key); + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this, chacha_key]() { m_account.encrypt_keys(chacha_key); m_account.decrypt_viewkey(chacha_key); }); + } + MINFO("Creating spend key..."); std::vector<crypto::secret_key> multisig_keys; rct::key spend_pkey, spend_skey; @@ -3551,6 +3809,7 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, MINFO("Creating multisig address..."); CHECK_AND_ASSERT_THROW_MES(m_account.make_multisig(view_skey, rct::rct2sk(spend_skey), rct::rct2pk(spend_pkey), multisig_keys), "Failed to create multisig wallet due to bad keys"); + memwipe(&spend_skey, sizeof(rct::key)); m_account_public_address = m_account.get_keys().m_account_address; m_watch_only = false; @@ -3568,23 +3827,12 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, m_multisig_signers = std::vector<crypto::public_key>(spend_keys.size() + 1, crypto::null_pkey); } - if (!m_wallet_file.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + // re-encrypt keys + keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); - if (boost::filesystem::exists(m_wallet_file + ".address.txt")) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (!m_wallet_file.empty()) store(); @@ -3650,6 +3898,17 @@ bool wallet2::finalize_multisig(const epee::wipeable_string &password, std::unor { CHECK_AND_ASSERT_THROW_MES(!pkeys.empty(), "empty pkeys"); + // keys are decrypted + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + { + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(password.data(), password.size(), chacha_key, m_kdf_rounds); + m_account.encrypt_viewkey(chacha_key); + m_account.decrypt_keys(chacha_key); + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this, chacha_key]() { m_account.encrypt_keys(chacha_key); m_account.decrypt_viewkey(chacha_key); }); + } + // add ours if not included crypto::public_key local_signer; CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(get_account().get_keys().m_spend_secret_key, local_signer), @@ -3672,17 +3931,10 @@ bool wallet2::finalize_multisig(const epee::wipeable_string &password, std::unor m_multisig_signers = signers; std::sort(m_multisig_signers.begin(), m_multisig_signers.end(), [](const crypto::public_key &e0, const crypto::public_key &e1){ return memcmp(&e0, &e1, sizeof(e0)); }); - if (!m_wallet_file.empty()) - { - bool r = store_keys(m_keys_file, password, false); - THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + // keys are encrypted again + keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); - if (boost::filesystem::exists(m_wallet_file + ".address.txt")) - { - r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_nettype)); - if(!r) MERROR("String with address text not saved"); - } - } + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); m_subaddresses.clear(); m_subaddress_labels.clear(); @@ -3980,7 +4232,12 @@ bool wallet2::check_connection(uint32_t *version, uint32_t timeout) bool wallet2::generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const { hw::device &hwdev = m_account.get_device(); - return hwdev.generate_chacha_key(m_account.get_keys(), key); + return hwdev.generate_chacha_key(m_account.get_keys(), key, m_kdf_rounds); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::generate_chacha_key_from_password(const epee::wipeable_string &pass, crypto::chacha_key &key) const +{ + crypto::generate_chacha_key(pass.data(), pass.size(), key, m_kdf_rounds); } //---------------------------------------------------------------------------------------------------- void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password) @@ -3991,17 +4248,19 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass boost::system::error_code e; bool exists = boost::filesystem::exists(m_keys_file, e); THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); - m_keys_file_locker.reset(new tools::file_locker(m_keys_file)); - THROW_WALLET_EXCEPTION_IF(!m_keys_file_locker->locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); + lock_keys_file(); + THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). - m_keys_file_locker.reset(); + unlock_keys_file(); if (!load_keys(m_keys_file, password)) { THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); } LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); - m_keys_file_locker.reset(new tools::file_locker(m_keys_file)); + lock_keys_file(); + + wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, password); //keys loaded ok! //try to load wallet file. but even if we failed, it is not big problem @@ -4024,11 +4283,9 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass r = ::serialization::parse_binary(buf, cache_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"'); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); std::string cache_data; cache_data.resize(cache_file_data.cache_data.size()); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cache_data[0]); try { std::stringstream iss; @@ -4036,11 +4293,13 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass boost::archive::portable_binary_iarchive ar(iss); ar >> *this; } - catch (...) + catch(...) { - crypto::chacha8(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); - try - { + // try with previous scheme: direct from keys + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + try { std::stringstream iss; iss << cache_data; boost::archive::portable_binary_iarchive ar(iss); @@ -4048,13 +4307,24 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass } catch (...) { - LOG_PRINT_L0("Failed to open portable binary, trying unportable"); - boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); - std::stringstream iss; - iss.str(""); - iss << cache_data; - boost::archive::binary_iarchive ar(iss); - ar >> *this; + crypto::chacha8(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + try + { + std::stringstream iss; + iss << cache_data; + boost::archive::portable_binary_iarchive ar(iss); + ar >> *this; + } + catch (...) + { + LOG_PRINT_L0("Failed to open portable binary, trying unportable"); + boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + std::stringstream iss; + iss.str(""); + iss << cache_data; + boost::archive::binary_iarchive ar(iss); + ar >> *this; + } } } } @@ -4206,12 +4476,10 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas wallet2::cache_file_data cache_file_data = boost::value_initialized<wallet2::cache_file_data>(); cache_file_data.cache_data = oss.str(); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); std::string cipher; cipher.resize(cache_file_data.cache_data.size()); cache_file_data.iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cipher[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cipher[0]); cache_file_data.cache_data = cipher; const std::string new_file = same_file ? m_wallet_file + ".new" : path; @@ -4477,12 +4745,7 @@ void wallet2::rescan_blockchain(bool refresh) { clear(); - cryptonote::block genesis; - generate_genesis(genesis); - crypto::hash genesis_hash = get_block_hash(genesis); - m_blockchain.push_back(genesis_hash); - m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); - add_subaddress_account(tr("Primary account")); + setup_new_blockchain(); if (refresh) this->refresh(false); @@ -4519,7 +4782,7 @@ bool wallet2::is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_heig uint64_t current_time = static_cast<uint64_t>(time(NULL)); // XXX: this needs to be fast, so we'd need to get the starting heights // from the daemon to be correct once voting kicks in - uint64_t v2height = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? (uint64_t)-1/*TODO*/ : 1009827; + uint64_t v2height = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? 32000 : 1009827; uint64_t leeway = block_height < v2height ? CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V1 : CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V2; if(current_time + leeway >= unlock_time) return true; @@ -4669,7 +4932,7 @@ size_t wallet2::pop_best_value(std::vector<size_t> &unused_indices, const std::v // returns: // direct return: amount of money found // modified reference: selected_transfers, a list of iterators/indices of input sources -uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const +uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const { uint64_t found_money = 0; selected_transfers.reserve(unused_transfers_indices.size()); @@ -4712,69 +4975,6 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo } //---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon) -{ - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx, trusted_daemon); -} -//---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon) -{ - cryptonote::transaction tx; - pending_tx ptx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx, trusted_daemon); -} - -namespace { -// split_amounts(vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -// -// split amount for each dst in dsts into num_splits parts -// and make num_splits new vector<crypt...> instances to hold these new amounts -std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts( - std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -{ - std::vector<std::vector<cryptonote::tx_destination_entry>> retVal; - - if (num_splits <= 1) - { - retVal.push_back(dsts); - return retVal; - } - - // for each split required - for (size_t i=0; i < num_splits; i++) - { - std::vector<cryptonote::tx_destination_entry> new_dsts; - - // for each destination - for (size_t j=0; j < dsts.size(); j++) - { - cryptonote::tx_destination_entry de; - uint64_t amount; - - amount = dsts[j].amount; - amount = amount / num_splits; - - // if last split, add remainder - if (i + 1 == num_splits) - { - amount += dsts[j].amount % num_splits; - } - - de.addr = dsts[j].addr; - de.amount = amount; - - new_dsts.push_back(de); - } - - retVal.push_back(new_dsts); - } - - return retVal; -} -} // anonymous namespace -//---------------------------------------------------------------------------------------------------- crypto::hash wallet2::get_payment_id(const pending_tx &ptx) const { std::vector<tx_extra_field> tx_extra_fields; @@ -5033,10 +5233,18 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofBulletproof; + for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) + if (proof.V.size() > 1) + range_proof_type = rct::RangeProofPaddedBulletproof; + } crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, sd.use_bulletproofs, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, range_proof_type, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -5449,7 +5657,15 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, sd.use_bulletproofs, &msout, false); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofBulletproof; + for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) + if (proof.V.size() > 1) + range_proof_type = rct::RangeProofPaddedBulletproof; + } + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -5549,9 +5765,18 @@ bool wallet2::sign_multisig_tx_from_file(const std::string &filename, std::vecto //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const { - static const uint64_t old_multipliers[3] = {1, 2, 3}; - static const uint64_t new_multipliers[3] = {1, 20, 166}; - static const uint64_t newer_multipliers[4] = {1, 4, 20, 166}; + static const struct + { + size_t count; + uint64_t multipliers[4]; + } + multipliers[] = + { + { 3, {1, 2, 3} }, + { 3, {1, 20, 166} }, + { 4, {1, 4, 20, 166} }, + { 4, {1, 5, 25, 1000} }, + }; if (fee_algorithm == -1) fee_algorithm = get_fee_algorithm(); @@ -5567,47 +5792,68 @@ uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const priority = 1; } + THROW_WALLET_EXCEPTION_IF(fee_algorithm < 0 || fee_algorithm > 3, error::invalid_priority); + // 1 to 3/4 are allowed as priorities - uint32_t max_priority = (fee_algorithm >= 2) ? 4 : 3; + const uint32_t max_priority = multipliers[fee_algorithm].count; if (priority >= 1 && priority <= max_priority) { - switch (fee_algorithm) - { - case 0: return old_multipliers[priority-1]; - case 1: return new_multipliers[priority-1]; - case 2: return newer_multipliers[priority-1]; - default: THROW_WALLET_EXCEPTION_IF (true, error::invalid_priority); - } + return multipliers[fee_algorithm].multipliers[priority-1]; } THROW_WALLET_EXCEPTION_IF (false, error::invalid_priority); return 1; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_dynamic_per_kb_fee_estimate() const +uint64_t wallet2::get_dynamic_base_fee_estimate() const { uint64_t fee; - boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_per_kb_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); + boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_base_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); if (!result) return fee; - LOG_PRINT_L1("Failed to query per kB fee, using " << print_money(FEE_PER_KB)); - return FEE_PER_KB; + const uint64_t base_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE) ? FEE_PER_BYTE : FEE_PER_KB; + LOG_PRINT_L1("Failed to query base fee, using " << print_money(base_fee)); + return base_fee; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_per_kb_fee() const +uint64_t wallet2::get_base_fee() const { if(m_light_wallet) - return m_light_wallet_per_kb_fee; + { + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + return m_light_wallet_per_kb_fee / 1024; + else + return m_light_wallet_per_kb_fee; + } bool use_dyn_fee = use_fork_rules(HF_VERSION_DYNAMIC_FEE, -720 * 1); if (!use_dyn_fee) return FEE_PER_KB; - return get_dynamic_per_kb_fee_estimate(); + return get_dynamic_base_fee_estimate(); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::get_fee_quantization_mask() const +{ + if(m_light_wallet) + { + return 1; // TODO + } + bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + if (!use_per_byte_fee) + return 1; + + uint64_t fee_quantization_mask; + boost::optional<std::string> result = m_node_rpc_proxy.get_fee_quantization_mask(fee_quantization_mask); + if (result) + return 1; + return fee_quantization_mask; } //---------------------------------------------------------------------------------------------------- int wallet2::get_fee_algorithm() const { - // changes at v3 and v5 + // changes at v3, v5, v8 + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0)) + return 3; if (use_fork_rules(5, 0)) return 2; if (use_fork_rules(3, -720 * 14)) @@ -5615,19 +5861,39 @@ int wallet2::get_fee_algorithm() const return 0; } //------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_min_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + if (use_fork_rules(7, 10)) + return 7; + if (use_fork_rules(6, 10)) + return 5; + if (use_fork_rules(2, 10)) + return 3; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_max_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ uint64_t wallet2::adjust_mixin(uint64_t mixin) const { - if (mixin < 6 && use_fork_rules(7, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 7, using 7"); - mixin = 6; - } - else if (mixin < 4 && use_fork_rules(6, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 6, using 5"); - mixin = 4; + const uint64_t min_ring_size = get_min_ring_size(); + if (mixin + 1 < min_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too low, using " << min_ring_size); + mixin = min_ring_size-1; } - else if (mixin < 2 && use_fork_rules(2, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 2, using 3"); - mixin = 2; + const uint64_t max_ring_size = get_max_ring_size(); + if (max_ring_size && mixin + 1 > max_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too high, using " << max_ring_size); + mixin = max_ring_size-1; } return mixin; } @@ -5639,7 +5905,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) try { // check if there's a backlog in the tx pool - const double fee_level = get_fee_multiplier(1) * get_per_kb_fee() * (12/(double)13) / (double)1024; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + const uint64_t base_fee = get_base_fee(); + const uint64_t fee_multiplier = get_fee_multiplier(1); + const double fee_level = fee_multiplier * base_fee * (use_per_byte_fee ? 1 : (12/(double)13 / (double)1024)); const std::vector<std::pair<uint64_t, uint64_t>> blocks = estimate_backlog({std::make_pair(fee_level, fee_level)}); if (blocks.size() != 1) { @@ -5653,10 +5922,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) } // get the current full reward zone - uint64_t block_size_limit = 0; - const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); throw_on_rpc_response_error(result, "get_info"); - const uint64_t full_reward_zone = block_size_limit / 2; + const uint64_t full_reward_zone = block_weight_limit / 2; // get the last N block headers and sum the block sizes const size_t N = 10; @@ -5680,14 +5949,14 @@ uint32_t wallet2::adjust_priority(uint32_t priority) MERROR("Bad blockheaders size"); return priority; } - size_t block_size_sum = 0; + size_t block_weight_sum = 0; for (const cryptonote::block_header_response &i : getbh_res.headers) { - block_size_sum += i.block_size; + block_weight_sum += i.block_weight; } // estimate how 'full' the last N blocks are - const size_t P = 100 * block_size_sum / (N * full_reward_zone); + const size_t P = 100 * block_weight_sum / (N * full_reward_zone); MINFO((boost::format("The last %d blocks fill roughly %d%% of the full reward zone.") % N % P).str()); if (P > 80) { @@ -5705,115 +5974,6 @@ uint32_t wallet2::adjust_priority(uint32_t priority) return priority; } //---------------------------------------------------------------------------------------------------- -// separated the call(s) to wallet2::transfer into their own function -// -// this function will make multiple calls to wallet2::transfer if multiple -// transactions will be required -std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) -{ - const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true, trusted_daemon); - - const uint64_t fee_per_kb = get_per_kb_fee(); - const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); - - // failsafe split attempt counter - size_t attempt_count = 0; - - for(attempt_count = 1; ;attempt_count++) - { - size_t num_tx = 0.5 + pow(1.7,attempt_count-1); - - auto split_values = split_amounts(dsts, num_tx); - - // Throw if split_amounts comes back with a vector of size different than it should - if (split_values.size() != num_tx) - { - throw std::runtime_error("Splitting transactions returned a number of potential tx not equal to what was requested"); - } - - std::vector<pending_tx> ptx_vector; - try - { - // for each new destination vector (i.e. for each new tx) - for (auto & dst_vector : split_values) - { - cryptonote::transaction tx; - pending_tx ptx; - - // loop until fee is met without increasing tx size to next KB boundary. - const size_t estimated_tx_size = estimate_tx_size(false, unused_transfers_indices.size(), fake_outs_count, dst_vector.size(), extra.size(), false); - uint64_t needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); - do - { - transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx, trusted_daemon); - auto txBlob = t_serializable_object_to_blob(ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - } while (ptx.fee < needed_fee); - - ptx_vector.push_back(ptx); - - // mark transfers to be used as "spent" - for(size_t idx: ptx.selected_transfers) - { - set_spent(idx, 0); - } - } - - // if we made it this far, we've selected our transactions. committing them will mark them spent, - // so this is a failsafe in case they don't go through - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - - } - - // if we made it this far, we're OK to actually send the transactions - return ptx_vector; - - } - // only catch this here, other exceptions need to pass through to the calling function - catch (const tools::error::tx_too_big& e) - { - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - if (attempt_count >= MAX_SPLIT_ATTEMPTS) - { - throw; - } - } - catch (...) - { - // in case of some other exception, make sure any tx in queue are marked unspent again - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - throw; - } - } -} - bool wallet2::set_ring_database(const std::string &filename) { m_ring_database = filename; @@ -5849,12 +6009,6 @@ crypto::chacha_key wallet2::get_ringdb_key() return *m_ringdb_key; } -void wallet2::clear_ringdb_key() -{ - MINFO("clearing ringdb key"); - m_ringdb_key = boost::none; -} - bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -5865,7 +6019,6 @@ bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transac bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) { - key_ref kref(*this); try { return add_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5874,7 +6027,6 @@ bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) { if (!m_ringdb) return false; - key_ref kref(*this); try { return m_ringdb->remove_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5912,7 +6064,6 @@ bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto:: bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs) { - key_ref kref(*this); try { return get_ring(get_ringdb_key(), key_image, outs); } catch (const std::exception &e) { return false; } } @@ -5922,7 +6073,6 @@ bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uin if (!m_ringdb) return false; - key_ref kref(*this); try { return m_ringdb->set_ring(get_ringdb_key(), key_image, outs, relative); } catch (const std::exception &e) { return false; } } @@ -5934,7 +6084,6 @@ bool wallet2::find_and_save_rings(bool force) if (!m_ringdb) return false; - key_ref kref(*this); COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); @@ -5997,7 +6146,7 @@ bool wallet2::find_and_save_rings(bool force) return true; } -bool wallet2::blackball_output(const crypto::public_key &output) +bool wallet2::blackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -6005,7 +6154,7 @@ bool wallet2::blackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add) +bool wallet2::set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add) { if (!m_ringdb) return false; @@ -6014,14 +6163,13 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out bool ret = true; if (!add) ret &= m_ringdb->clear_blackballs(); - for (const auto &output: outputs) - ret &= m_ringdb->blackball(output); + ret &= m_ringdb->blackball(outputs); return ret; } catch (const std::exception &e) { return false; } } -bool wallet2::unblackball_output(const crypto::public_key &output) +bool wallet2::unblackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -6029,7 +6177,7 @@ bool wallet2::unblackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::is_output_blackballed(const crypto::public_key &output) const +bool wallet2::is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const { if (!m_ringdb) return false; @@ -6037,6 +6185,33 @@ bool wallet2::is_output_blackballed(const crypto::public_key &output) const catch (const std::exception &e) { return false; } } +bool wallet2::lock_keys_file() +{ + if (m_keys_file_locker) + { + MDEBUG(m_keys_file << " is already locked."); + return false; + } + m_keys_file_locker.reset(new tools::file_locker(m_keys_file)); + return true; +} + +bool wallet2::unlock_keys_file() +{ + if (!m_keys_file_locker) + { + MDEBUG(m_keys_file << " is already unlocked."); + return false; + } + m_keys_file_locker.reset(); + return true; +} + +bool wallet2::is_keys_file_locked() const +{ + return m_keys_file_locker->locked(); +} + bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const { if (!unlocked) // don't add locked outs @@ -6047,8 +6222,8 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; - if (is_output_blackballed(output_public_key)) // don't add blackballed outputs - return false; +// if (is_output_blackballed(output_public_key)) // don't add blackballed outputs +// return false; outs.back().push_back(item); return true; } @@ -6177,22 +6352,42 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> bool is_shortly_after_segregation_fork = height >= segregation_fork_height && height < segregation_fork_height + SEGREGATION_FORK_VICINITY; bool is_after_segregation_fork = height >= segregation_fork_height; + // if we have at least one rct out, get the distribution, or fall back to the previous system + uint64_t rct_start_height; + std::vector<uint64_t> rct_offsets; + bool has_rct = false; + for (size_t idx: selected_transfers) + if (m_transfers[idx].is_rct()) + { has_rct = true; break; } + const bool has_rct_distribution = has_rct && get_rct_distribution(rct_start_height, rct_offsets); + if (has_rct_distribution) + { + // check we're clear enough of rct start, to avoid corner cases below + THROW_WALLET_EXCEPTION_IF(rct_offsets.size() <= CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, + error::get_output_distribution, "Not enough rct outputs"); + } + // get histogram for the amounts we need cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); - m_daemon_rpc_mutex.lock(); + // request histogram for all outputs, except 0 if we have the rct distribution for(size_t idx: selected_transfers) - req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); - std::sort(req_t.amounts.begin(), req_t.amounts.end()); - auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); - req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); - req_t.unlocked = true; - req_t.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); - THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + if (!m_transfers[idx].is_rct() || !has_rct_distribution) + req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); + if (!req_t.amounts.empty()) + { + std::sort(req_t.amounts.begin(), req_t.amounts.end()); + auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); + req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); + req_t.unlocked = true; + req_t.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + } // if we want to segregate fake outs pre or post fork, get distribution std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> segregation_limit; @@ -6208,6 +6403,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> req_t.from_height = std::max<uint64_t>(segregation_fork_height, RECENT_OUTPUT_BLOCKS) - RECENT_OUTPUT_BLOCKS; req_t.to_height = segregation_fork_height + 1; req_t.cumulative = true; + req_t.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout * 1000); m_daemon_rpc_mutex.unlock(); @@ -6248,6 +6444,36 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> COMMAND_RPC_GET_OUTPUTS_BIN::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_OUTPUTS_BIN::response daemon_resp = AUTO_VAL_INIT(daemon_resp); + struct gamma_engine + { + typedef uint64_t result_type; + static constexpr result_type min() { return 0; } + static constexpr result_type max() { return std::numeric_limits<result_type>::max(); } + result_type operator()() { return crypto::rand<result_type>(); } + } engine; + static const double shape = 19.28/*16.94*/; + //static const double shape = m_testnet ? 17.02 : 17.28; + static const double scale = 1/1.61; + std::gamma_distribution<double> gamma(shape, scale); + auto pick_gamma = [&]() + { + double x = gamma(engine); + x = exp(x); + uint64_t block_offset = x / DIFFICULTY_TARGET_V2; // this assumes constant target over the whole rct range + if (block_offset >= rct_offsets.size() - 1) + return std::numeric_limits<uint64_t>::max(); // bad pick + block_offset = rct_offsets.size() - 2 - block_offset; + THROW_WALLET_EXCEPTION_IF(block_offset >= rct_offsets.size() - 1, error::wallet_internal_error, "Bad offset calculation"); + THROW_WALLET_EXCEPTION_IF(rct_offsets[block_offset + 1] < rct_offsets[block_offset], + error::get_output_distribution, "Decreasing offsets in rct distribution: " + + std::to_string(block_offset) + ": " + std::to_string(rct_offsets[block_offset]) + ", " + + std::to_string(block_offset + 1) + ": " + std::to_string(rct_offsets[block_offset + 1])); + uint64_t n_rct = rct_offsets[block_offset + 1] - rct_offsets[block_offset]; + if (n_rct == 0) + return rct_offsets[block_offset] ? rct_offsets[block_offset] - 1 : 0; + return rct_offsets[block_offset] + crypto::rand<uint64_t>() % n_rct; + }; + size_t num_selected_transfers = 0; for(size_t idx: selected_transfers) { @@ -6258,6 +6484,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // request more for rct in base recent (locked) coinbases are picked, since they're locked for longer size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0); size_t start = req.outputs.size(); + bool use_histogram = amount != 0 || !has_rct_distribution; const bool output_is_pre_fork = td.m_block_height < segregation_fork_height; uint64_t num_outs = 0, num_recent_outs = 0; @@ -6313,26 +6540,41 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> num_post_fork_outs = num_outs - segregation_limit[amount].first; } - LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); - THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, - "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); - THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, - "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + if (use_histogram) + { + LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); + THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, + "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); + THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, + "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + } + else + { + // the base offset of the first rct output in the first unlocked block (or the one to be if there's none) + num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; + LOG_PRINT_L1("" << num_outs << " unlocked rct outputs"); + THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, + "histogram reports no unlocked rct outputs, not even ours"); + } - // how many fake outs to draw on a pre-fork triangular distribution + // how many fake outs to draw on a pre-fork distribution size_t pre_fork_outputs_count = requested_outputs_count * pre_fork_num_out_ratio; size_t post_fork_outputs_count = requested_outputs_count * post_fork_num_out_ratio; // how many fake outs to draw otherwise size_t normal_output_count = requested_outputs_count - pre_fork_outputs_count - post_fork_outputs_count; - // X% of those outs are to be taken from recent outputs - size_t recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; - if (recent_outputs_count == 0) - recent_outputs_count = 1; // ensure we have at least one, if possible - if (recent_outputs_count > num_recent_outs) - recent_outputs_count = num_recent_outs; - if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) - --recent_outputs_count; // if the real out is recent, pick one less recent fake out + size_t recent_outputs_count = 0; + if (use_histogram) + { + // X% of those outs are to be taken from recent outputs + recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; + if (recent_outputs_count == 0) + recent_outputs_count = 1; // ensure we have at least one, if possible + if (recent_outputs_count > num_recent_outs) + recent_outputs_count = num_recent_outs; + if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) + --recent_outputs_count; // if the real out is recent, pick one less recent fake out + } LOG_PRINT_L1("Fake output makeup: " << requested_outputs_count << " requested: " << recent_outputs_count << " recent, " << pre_fork_outputs_count << " pre-fork, " << post_fork_outputs_count << " post-fork, " << (requested_outputs_count - recent_outputs_count - pre_fork_outputs_count - post_fork_outputs_count) << " full-chain"); @@ -6412,7 +6654,26 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> uint64_t i; const char *type = ""; - if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with + if (amount == 0 && has_rct_distribution) + { + // gamma distribution + if (num_found -1 < recent_outputs_count + pre_fork_outputs_count) + { + do i = pick_gamma(); while (i >= segregation_limit[amount].first); + type = "pre-fork gamma"; + } + else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count + post_fork_outputs_count) + { + do i = pick_gamma(); while (i < segregation_limit[amount].first || i >= num_outs); + type = "post-fork gamma"; + } + else + { + do i = pick_gamma(); while (i >= num_outs); + type = "gamma"; + } + } + else if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with { // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); @@ -6459,6 +6720,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (seen_indices.count(i)) continue; + if (is_output_blackballed(std::make_pair(amount, i))) // don't add blackballed outputs + continue; seen_indices.emplace(i); LOG_PRINT_L2("picking " << i << " as " << type); @@ -6477,11 +6740,11 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // get the keys for those m_daemon_rpc_mutex.lock(); - r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, daemon_resp.status); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); @@ -6615,7 +6878,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer: starting with fee " << print_money (needed_money)); @@ -6717,10 +6980,10 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBulletproof, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); std::string key_images; bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool @@ -6766,13 +7029,13 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type) { using namespace cryptonote; // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer_selected_rct: starting with fee " << print_money (needed_money)); LOG_PRINT_L2("selected transfers: " << strjoin(selected_transfers, " ")); @@ -6922,10 +7185,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); auto sources_copy = sources; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, bulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, range_proof_type, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); // work out the permutation done on sources std::vector<size_t> ins_order; @@ -6959,7 +7222,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry THROW_WALLET_EXCEPTION_IF(selected_transfers.size() != sources.size(), error::wallet_internal_error, "mismatched selected_transfers and sources sixes"); for(size_t idx: selected_transfers) { - cryptonote::tx_source_entry& src = sources[src_idx]; + cryptonote::tx_source_entry& src = sources_copy[src_idx]; src.multisig_kLRki = get_multisig_composite_kLRki(idx, multisig_signers[signer_index], used_L, new_used_L); ++src_idx; } @@ -6967,10 +7230,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry LOG_PRINT_L2("Creating supplementary multisig transaction"); cryptonote::transaction ms_tx; auto sources_copy_copy = sources_copy; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, bulletproof, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, range_proof_type, &msout, false); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_prefix_hash(ms_tx) != prefix_hash, error::wallet_internal_error, "Multisig txes do not share prefix"); multisig_sigs.push_back({ms_tx.rct_signatures, multisig_signers[signer_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); @@ -7437,6 +7700,7 @@ void wallet2::light_wallet_get_address_txs() payment.m_block_height = t.height; payment.m_unlock_time = t.unlock_time; payment.m_timestamp = t.timestamp; + payment.m_coinbase = t.coinbase; if (t.mempool) { if (std::find(unconfirmed_payments_txs.begin(), unconfirmed_payments_txs.end(), tx_hash) == unconfirmed_payments_txs.end()) { @@ -7617,7 +7881,7 @@ bool wallet2::light_wallet_key_image_is_ours(const crypto::key_image& key_image, // This system allows for sending (almost) the entire balance, since it does // not generate spurious change in all txes, thus decreasing the instantaneous // usable balance. -std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -7637,11 +7901,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; uint64_t needed_fee; std::vector<std::vector<tools::wallet2::get_outs_entry>> outs; - TX() : bytes(0), needed_fee(0) {} + TX() : weight(0), needed_fee(0) {} void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) @@ -7669,12 +7933,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<TX> txes; bool adding_fee; // true if new outputs go towards fee, rather than destinations uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); const bool use_rct = use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); @@ -7705,6 +7972,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // early out if we know we can't make it anyway // we could also check for being within FEE_PER_KB, but if the fee calculation // ever changes, this might be missed, so let this go through + const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)) / 1024; uint64_t balance_subtotal = 0; uint64_t unlocked_balance_subtotal = 0; for (uint32_t index_minor : subaddr_indices) @@ -7712,21 +7980,21 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp balance_subtotal += balance_per_subaddr[index_minor]; unlocked_balance_subtotal += unlocked_balance_per_subaddr[index_minor]; } - THROW_WALLET_EXCEPTION_IF(needed_money > balance_subtotal, error::not_enough_money, + THROW_WALLET_EXCEPTION_IF(needed_money + min_fee > balance_subtotal, error::not_enough_money, balance_subtotal, needed_money, 0); // first check overall balance is enough, then unlocked one, so we throw distinct exceptions - THROW_WALLET_EXCEPTION_IF(needed_money > unlocked_balance_subtotal, error::not_enough_unlocked_money, + THROW_WALLET_EXCEPTION_IF(needed_money + min_fee > unlocked_balance_subtotal, error::not_enough_unlocked_money, unlocked_balance_subtotal, needed_money, 0); for (uint32_t i : subaddr_indices) LOG_PRINT_L2("Candidate subaddress index for spending: " << i); // determine threshold for fractional amount - const size_t tx_size_one_ring = estimate_tx_size(use_rct, 1, fake_outs_count, 2, 0, bulletproof); - const size_t tx_size_two_rings = estimate_tx_size(use_rct, 2, fake_outs_count, 2, 0, bulletproof); - THROW_WALLET_EXCEPTION_IF(tx_size_one_ring > tx_size_two_rings, error::wallet_internal_error, "Estimated tx size with 1 input is larger than with 2 inputs!"); - const size_t tx_size_per_ring = tx_size_two_rings - tx_size_one_ring; - const uint64_t fractional_threshold = (fee_multiplier * fee_per_kb * tx_size_per_ring) / 1024; + const size_t tx_weight_one_ring = estimate_tx_weight(use_rct, 1, fake_outs_count, 2, 0, bulletproof); + const size_t tx_weight_two_rings = estimate_tx_weight(use_rct, 2, fake_outs_count, 2, 0, bulletproof); + THROW_WALLET_EXCEPTION_IF(tx_weight_one_ring > tx_weight_two_rings, error::wallet_internal_error, "Estimated tx weight with 1 input is larger than with 2 inputs!"); + const size_t tx_weight_per_ring = tx_weight_two_rings - tx_weight_one_ring; + const uint64_t fractional_threshold = (fee_multiplier * base_fee * tx_weight_per_ring) / (use_per_byte_fee ? 1 : 1024); // gather all dust and non-dust outputs belonging to specified subaddresses size_t num_nondust_outputs = 0; @@ -7819,7 +8087,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp { // this is used to build a tx that's 1 or 2 inputs, and 2 outputs, which // will get us a known fee. - uint64_t estimated_fee = calculate_fee(fee_per_kb, estimate_rct_tx_size(2, fake_outs_count, 2, extra.size(), bulletproof), fee_multiplier); + uint64_t estimated_fee = estimate_fee(use_per_byte_fee, use_rct, 2, fake_outs_count, 2, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); preferred_inputs = pick_preferred_rct_inputs(needed_money + estimated_fee, subaddr_account, subaddr_indices); if (!preferred_inputs.empty()) { @@ -7861,7 +8129,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while ((!dsts.empty() && dsts[0].amount > 0) || adding_fee || !preferred_inputs.empty() || should_pick_a_second_output(use_rct, txes.back().selected_transfers.size(), *unused_transfers_indices, *unused_dust_indices)) { TX &tx = txes.back(); - LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size()); + LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size() << ", tx.dsts.size() " << tx.dsts.size()); LOG_PRINT_L2("unused_transfers_indices: " << strjoin(*unused_transfers_indices, " ")); LOG_PRINT_L2("unused_dust_indices: " << strjoin(*unused_dust_indices, " ")); LOG_PRINT_L2("dsts size " << dsts.size() << ", first " << (dsts.empty() ? "-" : cryptonote::print_money(dsts[0].amount))); @@ -7924,7 +8192,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) + while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << @@ -7936,7 +8204,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp ++original_output_index; } - if (available_amount > 0 && !dsts.empty() && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) { + if (available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); @@ -7948,7 +8216,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); + << upper_transaction_weight_limit); bool try_tx = false; // if we have preferred picks, but haven't yet used all of them, continue if (preferred_inputs.empty()) @@ -7960,8 +8228,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - try_tx = dsts.empty() || (estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + try_tx = dsts.empty() || (estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); } } @@ -7969,8 +8237,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_multiplier); uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); @@ -7987,14 +8254,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.selected_transfers.size() << " inputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); available_for_fee = test_ptx.fee + test_ptx.change_dts.amount + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0); - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0) @@ -8030,22 +8297,22 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while (needed_fee > test_ptx.fee) { if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; @@ -8103,7 +8370,7 @@ skip_tx: extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ - bulletproof); + range_proof_type); } else { transfer_selected(tx.dsts, tx.selected_transfers, @@ -8120,7 +8387,7 @@ skip_tx: auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -8131,7 +8398,7 @@ skip_tx: for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8141,7 +8408,7 @@ skip_tx: return ptx_vector; } -std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -8192,10 +8459,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -8213,10 +8480,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypt break; } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -8229,21 +8496,24 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; uint64_t needed_fee; std::vector<std::vector<get_outs_entry>> outs; - TX() : bytes(0), needed_fee(0) {} + TX() : weight(0), needed_fee(0) {} }; std::vector<TX> txes; uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); std::vector<std::vector<get_outs_entry>> outs; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const uint64_t fee_per_kb = get_per_kb_fee(); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); LOG_PRINT_L2("Starting with " << unused_transfers_indices.size() << " non-dust outputs and " << unused_dust_indices.size() << " dust outputs"); @@ -8265,7 +8535,25 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // get a random unspent output and use it to pay next chunk. We try to alternate // dust and non dust to ensure we never get with only dust, from which we might // get a tx that can't pay for itself - size_t idx = unused_transfers_indices.empty() ? pop_best_value(unused_dust_indices, tx.selected_transfers) : unused_dust_indices.empty() ? pop_best_value(unused_transfers_indices, tx.selected_transfers) : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_per_kb * fee_multiplier * (upper_transaction_size_limit + 1023) / 1024) ? pop_best_value(unused_dust_indices, tx.selected_transfers) : pop_best_value(unused_transfers_indices, tx.selected_transfers); + uint64_t fee_dust_threshold; + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + { + const uint64_t estimated_tx_weight_with_one_extra_output = estimate_tx_weight(use_rct, tx.selected_transfers.size() + 1, fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + fee_dust_threshold = calculate_fee_from_weight(base_fee, estimated_tx_weight_with_one_extra_output, fee_multiplier, fee_quantization_mask); + } + else + { + fee_dust_threshold = base_fee * fee_multiplier * (upper_transaction_weight_limit + 1023) / 1024; + } + + size_t idx = + unused_transfers_indices.empty() + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : unused_dust_indices.empty() + ? pop_best_value(unused_transfers_indices, tx.selected_transfers) + : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_dust_threshold) + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : pop_best_value(unused_transfers_indices, tx.selected_transfers); const transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Picking output " << idx << ", amount " << print_money(td.amount())); @@ -8280,16 +8568,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 1, extra.size(), bulletproof); - bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + << upper_transaction_weight_limit); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 2, extra.size(), bulletproof); + bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); if (try_tx) { cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); tx.dsts.push_back(tx_destination_entry(1, address, is_subaddress)); @@ -8297,14 +8584,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.selected_transfers.size() << " outputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); available_for_fee = test_ptx.fee + test_ptx.dests[0].amount + test_ptx.change_dts.amount; - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); THROW_WALLET_EXCEPTION_IF(needed_fee > available_for_fee, error::wallet_internal_error, "Transaction cannot pay for itself"); @@ -8314,22 +8601,22 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.dsts[0].amount = available_for_fee - needed_fee; if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } while (needed_fee > test_ptx.fee); - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; @@ -8353,7 +8640,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton pending_tx test_ptx; if (use_rct) { transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); } else { transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -8361,7 +8648,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -8372,7 +8659,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8399,7 +8686,7 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); throw_on_rpc_response_error(result, "get_hard_fork_info"); - bool close_enough = height >= earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand + bool close_enough = height >= earliest_height - early_blocks; // start using the rules that many blocks beforehand if (close_enough) LOG_PRINT_L2("Using v" << (unsigned)version << " rules"); else @@ -8407,12 +8694,15 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const return close_enough; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_upper_transaction_size_limit() const +uint64_t wallet2::get_upper_transaction_weight_limit() const { - if (m_upper_transaction_size_limit > 0) - return m_upper_transaction_size_limit; + if (m_upper_transaction_weight_limit > 0) + return m_upper_transaction_weight_limit; uint64_t full_reward_zone = use_fork_rules(5, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : use_fork_rules(2, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; - return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + if (use_fork_rules(8, 10)) + return full_reward_zone / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } //---------------------------------------------------------------------------------------------------- std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const @@ -8450,12 +8740,12 @@ std::vector<uint64_t> wallet2::get_unspent_amounts_vector() const return vector; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon) +std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct) { cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - if (trusted_daemon) + if (is_trusted_daemon()) req_t.amounts = get_unspent_amounts_vector(); req_t.min_count = count; req_t.max_count = 0; @@ -8516,30 +8806,28 @@ const wallet2::transfer_details &wallet2::get_transfer_details(size_t idx) const return m_transfers[idx]; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_unmixable_outputs() { - // request all outputs with less than 3 instances - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, false, true, false, trusted_daemon); + // request all outputs with less instances than the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), false, true, false); } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_mixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_mixable_outputs() { - // request all outputs with at least 3 instances, so we can use mixin 2 with - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, true, true, true, trusted_daemon); + // request all outputs with at least as many instances as the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), true, true, true); } //---------------------------------------------------------------------------------------------------- -std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions() { // From hard fork 1, we don't consider small amounts to be dust anymore const bool hf1_rules = use_fork_rules(2, 10); // first hard fork has version 2 tx_dust_policy dust_policy(hf1_rules ? 0 : ::config::DEFAULT_DUST_THRESHOLD); - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); // may throw - std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); size_t num_dust_outputs = unmixable_outputs.size(); if (num_dust_outputs == 0) @@ -8551,19 +8839,19 @@ std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bo std::vector<size_t> unmixable_transfer_outputs, unmixable_dust_outputs; for (auto n: unmixable_outputs) { - if (m_transfers[n].amount() < fee_per_kb) + if (m_transfers[n].amount() < base_fee) unmixable_dust_outputs.push_back(n); else unmixable_transfer_outputs.push_back(n); } - return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>(), trusted_daemon); + return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>()); } //---------------------------------------------------------------------------------------------------- -void wallet2::discard_unmixable_outputs(bool trusted_daemon) +void wallet2::discard_unmixable_outputs() { // may throw - std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); for (size_t idx : unmixable_outputs) { m_transfers[idx].m_spent = true; @@ -8583,6 +8871,54 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s return true; } //---------------------------------------------------------------------------------------------------- +void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys) +{ + // fetch tx from daemon and check if secret keys agree with corresponding public keys + COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = false; + COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); + bool r; + { + const boost::lock_guard<boost::mutex> lock{m_daemon_rpc_mutex}; + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + } + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, + "daemon returned wrong response for gettransactions, wrong txs count = " + + std::to_string(res.txs.size()) + ", expected 1"); + cryptonote::blobdata bd; + THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; + crypto::hash tx_hash, tx_prefix_hash; + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + std::vector<tx_extra_field> tx_extra_fields; + THROW_WALLET_EXCEPTION_IF(!parse_tx_extra(tx.extra, tx_extra_fields), error::wallet_internal_error, "Transaction extra has unsupported format"); + tx_extra_pub_key pub_key_field; + bool found = false; + size_t index = 0; + while (find_tx_extra_field_by_type(tx_extra_fields, pub_key_field, index++)) + { + crypto::public_key calculated_pub_key; + crypto::secret_key_to_public_key(tx_key, calculated_pub_key); + if (calculated_pub_key == pub_key_field.pub_key) + { + found = true; + break; + } + } + THROW_WALLET_EXCEPTION_IF(!found, error::wallet_internal_error, "Given tx secret key doesn't agree with the tx public key in the blockchain"); + tx_extra_additional_pub_keys additional_tx_pub_keys; + find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys); + THROW_WALLET_EXCEPTION_IF(additional_tx_keys.size() != additional_tx_pub_keys.data.size(), error::wallet_internal_error, "The number of additional tx secret keys doesn't agree with the number of additional tx public keys in the blockchain" ); + m_tx_keys.insert(std::make_pair(txid, tx_key)); + m_additional_tx_keys.insert(std::make_pair(txid, additional_tx_keys)); +} +//---------------------------------------------------------------------------------------------------- std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string &message) { THROW_WALLET_EXCEPTION_IF(m_watch_only, error::wallet_internal_error, @@ -8896,6 +9232,8 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1)); const rct::key C = tx.rct_signatures.outPk[n].mask; rct::key Ctmp; + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.mask.bytes) != 0, error::wallet_internal_error, "Bad ECDH input mask"); + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.amount.bytes) != 0, error::wallet_internal_error, "Bad ECDH input amount"); rct::addKeys2(Ctmp, ecdh_info.mask, ecdh_info.amount, rct::H); if (rct::equalKeys(C, Ctmp)) amount = rct::h2d(ecdh_info.amount); @@ -9183,6 +9521,7 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, if (account_minreserve) { + THROW_WALLET_EXCEPTION_IF(account_minreserve->second == 0, error::wallet_internal_error, "Proved amount must be greater than 0"); // minimize the number of outputs included in the proof, by only picking the N largest outputs that can cover the requested min reserve amount std::sort(selected_transfers.begin(), selected_transfers.end(), [&](const size_t a, const size_t b) { return m_transfers[a].amount() > m_transfers[b].amount(); }); @@ -9466,7 +9805,7 @@ uint64_t wallet2::get_approximate_blockchain_height() const // Calculated blockchain height uint64_t approx_blockchain_height = fork_block + (time(NULL) - fork_time)/seconds_per_block; // testnet got some huge rollbacks, so the estimation is way off - static const uint64_t approximate_testnet_rolled_back_blocks = 148540; + static const uint64_t approximate_testnet_rolled_back_blocks = 303967; if (m_nettype == TESTNET && approx_blockchain_height > approximate_testnet_rolled_back_blocks) approx_blockchain_height -= approximate_testnet_rolled_back_blocks; LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height); @@ -9870,6 +10209,17 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag std::unordered_set<crypto::hash> spent_txids; // For each spent key image, search for a tx in m_transfers that uses it as input. std::vector<size_t> swept_transfers; // If such a spending tx wasn't found in m_transfers, this means the spending tx // was created by sweep_all, so we can't know the spent height and other detailed info. + std::unordered_map<crypto::key_image, crypto::hash> spent_key_images; + + for (const transfer_details &td: m_transfers) + { + for (const cryptonote::txin_v& in : td.m_tx.vin) + { + if (in.type() == typeid(cryptonote::txin_to_key)) + spent_key_images.insert(std::make_pair(boost::get<cryptonote::txin_to_key>(in).k_image, td.m_txid)); + } + } + for(size_t i = 0; i < signed_key_images.size(); ++i) { transfer_details &td = m_transfers[i]; @@ -9883,28 +10233,11 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag if (i < daemon_resp.spent_status.size() && daemon_resp.spent_status[i] == COMMAND_RPC_IS_KEY_IMAGE_SPENT::SPENT_IN_BLOCKCHAIN) { - bool is_spent_tx_found = false; - for (auto it = m_transfers.rbegin(); &(*it) != &td; ++it) - { - bool is_spent_tx = false; - for(const cryptonote::txin_v& in : it->m_tx.vin) - { - if(in.type() == typeid(cryptonote::txin_to_key) && td.m_key_image == boost::get<cryptonote::txin_to_key>(in).k_image) - { - is_spent_tx = true; - break; - } - } - if (is_spent_tx) - { - is_spent_tx_found = true; - spent_txids.insert(it->m_txid); - break; - } - } - - if (!is_spent_tx_found) + const std::unordered_map<crypto::key_image, crypto::hash>::const_iterator skii = spent_key_images.find(td.m_key_image); + if (skii == spent_key_images.end()) swept_transfers.push_back(i); + else + spent_txids.insert(skii->second); } } MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); @@ -10027,11 +10360,10 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag { const transfer_details& td = m_transfers[n]; confirmed_transfer_details pd; - pd.m_change = (uint64_t)-1; // cahnge is unknown + pd.m_change = (uint64_t)-1; // change is unknown pd.m_amount_in = pd.m_amount_out = td.amount(); // fee is unknown - std::string err; - pd.m_block_height = get_daemon_blockchain_height(err); // spent block height is unknown, so hypothetically set to the highest - crypto::hash spent_txid = crypto::rand<crypto::hash>(); // spent txid is unknown, so hypothetically set to random + pd.m_block_height = 0; // spent block height is unknown + const crypto::hash &spent_txid = crypto::null_hash; // spent txid is unknown m_confirmed_txs.insert(std::make_pair(spent_txid, pd)); } } @@ -10143,18 +10475,20 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout[td.m_internal_output_index].target.type() != typeid(cryptonote::txout_to_key), + error::wallet_internal_error, "Unsupported output type"); const crypto::public_key& out_key = boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key; bool r = cryptonote::generate_key_image_helper(m_account.get_keys(), m_subaddresses, out_key, tx_pub_key, additional_tx_pub_keys, td.m_internal_output_index, in_ephemeral, td.m_key_image, m_account.get_device()); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; td.m_key_image_partial = false; - THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key, + THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i)); m_key_images[td.m_key_image] = m_transfers.size(); m_pub_keys[td.get_public_key()] = m_transfers.size(); - m_transfers.push_back(td); + m_transfers.push_back(std::move(td)); } return m_transfers.size(); @@ -10493,23 +10827,29 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) m_multisig_rescan_info = &info; try { + refresh(false); } - catch (...) {} + catch (...) + { + m_multisig_rescan_info = NULL; + m_multisig_rescan_k = NULL; + throw; + } m_multisig_rescan_info = NULL; m_multisig_rescan_k = NULL; return n_outputs; } //---------------------------------------------------------------------------------------------------- -std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated) const +std::string wallet2::encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated) const { crypto::chacha_key key; - crypto::generate_chacha_key(&skey, sizeof(skey), key); + crypto::generate_chacha_key(&skey, sizeof(skey), key, m_kdf_rounds); std::string ciphertext; crypto::chacha_iv iv = crypto::rand<crypto::chacha_iv>(); - ciphertext.resize(plaintext.size() + sizeof(iv) + (authenticated ? sizeof(crypto::signature) : 0)); - crypto::chacha20(plaintext.data(), plaintext.size(), key, iv, &ciphertext[sizeof(iv)]); + ciphertext.resize(len + sizeof(iv) + (authenticated ? sizeof(crypto::signature) : 0)); + crypto::chacha20(plaintext, len, key, iv, &ciphertext[sizeof(iv)]); memcpy(&ciphertext[0], &iv, sizeof(iv)); if (authenticated) { @@ -10523,22 +10863,36 @@ std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_ return ciphertext; } //---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const epee::span<char> &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const epee::wipeable_string &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- std::string wallet2::encrypt_with_view_secret_key(const std::string &plaintext, bool authenticated) const { return encrypt(plaintext, get_account().get_keys().m_view_secret_key, authenticated); } //---------------------------------------------------------------------------------------------------- -std::string wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const +template<typename T> +T wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const { const size_t prefix_size = sizeof(chacha_iv) + (authenticated ? sizeof(crypto::signature) : 0); THROW_WALLET_EXCEPTION_IF(ciphertext.size() < prefix_size, error::wallet_internal_error, "Unexpected ciphertext size"); crypto::chacha_key key; - crypto::generate_chacha_key(&skey, sizeof(skey), key); + crypto::generate_chacha_key(&skey, sizeof(skey), key, m_kdf_rounds); const crypto::chacha_iv &iv = *(const crypto::chacha_iv*)&ciphertext[0]; - std::string plaintext; - plaintext.resize(ciphertext.size() - prefix_size); if (authenticated) { crypto::hash hash; @@ -10549,10 +10903,14 @@ std::string wallet2::decrypt(const std::string &ciphertext, const crypto::secret THROW_WALLET_EXCEPTION_IF(!crypto::check_signature(hash, pkey, signature), error::wallet_internal_error, "Failed to authenticate ciphertext"); } - crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - prefix_size, key, iv, &plaintext[0]); - return plaintext; + std::unique_ptr<char[]> buffer{new char[ciphertext.size() - prefix_size]}; + auto wiper = epee::misc_utils::create_scope_leave_handler([&]() { memwipe(buffer.get(), ciphertext.size() - prefix_size); }); + crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - prefix_size, key, iv, buffer.get()); + return T(buffer.get(), ciphertext.size() - prefix_size); } //---------------------------------------------------------------------------------------------------- +template epee::wipeable_string wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const; +//---------------------------------------------------------------------------------------------------- std::string wallet2::decrypt_with_view_secret_key(const std::string &ciphertext, bool authenticated) const { return decrypt(ciphertext, get_account().get_keys().m_view_secret_key, authenticated); @@ -10810,45 +11168,46 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - uint64_t block_size_limit = 0; - const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); throw_on_rpc_response_error(result, "get_info"); - uint64_t full_reward_zone = block_size_limit / 2; + uint64_t full_reward_zone = block_weight_limit / 2; + THROW_WALLET_EXCEPTION_IF(full_reward_zone == 0, error::wallet_internal_error, "Invalid block weight limit from daemon"); std::vector<std::pair<uint64_t, uint64_t>> blocks; for (const auto &fee_level: fee_levels) { const double our_fee_byte_min = fee_level.first; const double our_fee_byte_max = fee_level.second; - uint64_t priority_size_min = 0, priority_size_max = 0; + uint64_t priority_weight_min = 0, priority_weight_max = 0; for (const auto &i: res.backlog) { - if (i.blob_size == 0) + if (i.weight == 0) { - MWARNING("Got 0 sized blob from txpool, ignored"); + MWARNING("Got 0 weight tx from txpool, ignored"); continue; } - double this_fee_byte = i.fee / (double)i.blob_size; + double this_fee_byte = i.fee / (double)i.weight; if (this_fee_byte >= our_fee_byte_min) - priority_size_min += i.blob_size; + priority_weight_min += i.weight; if (this_fee_byte >= our_fee_byte_max) - priority_size_max += i.blob_size; + priority_weight_max += i.weight; } - uint64_t nblocks_min = priority_size_min / full_reward_zone; - uint64_t nblocks_max = priority_size_max / full_reward_zone; - MDEBUG("estimate_backlog: priority_size " << priority_size_min << " - " << priority_size_max << " for " + uint64_t nblocks_min = priority_weight_min / full_reward_zone; + uint64_t nblocks_max = priority_weight_max / full_reward_zone; + MDEBUG("estimate_backlog: priority_weight " << priority_weight_min << " - " << priority_weight_max << " for " << our_fee_byte_min << " - " << our_fee_byte_max << " piconero byte fee, " - << nblocks_min << " - " << nblocks_max << " blocks at block size " << full_reward_zone); + << nblocks_min << " - " << nblocks_max << " blocks at block weight " << full_reward_zone); blocks.push_back(std::make_pair(nblocks_min, nblocks_max)); } return blocks; } //---------------------------------------------------------------------------------------------------- -std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees) +std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees) { - THROW_WALLET_EXCEPTION_IF(min_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); - THROW_WALLET_EXCEPTION_IF(max_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(min_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(max_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); for (uint64_t fee: fees) { THROW_WALLET_EXCEPTION_IF(fee == 0, error::wallet_internal_error, "Invalid 0 fee"); @@ -10856,7 +11215,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t mi std::vector<std::pair<double, double>> fee_levels; for (uint64_t fee: fees) { - double our_fee_byte_min = fee / (double)min_blob_size, our_fee_byte_max = fee / (double)max_blob_size; + double our_fee_byte_min = fee / (double)min_tx_weight, our_fee_byte_max = fee / (double)max_tx_weight; fee_levels.emplace_back(our_fee_byte_min, our_fee_byte_max); } return estimate_backlog(fee_levels); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index d4d76e66c..c30ca1d85 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -67,6 +67,19 @@ class Serialization_portability_wallet_Test; namespace tools { class ringdb; + class wallet2; + + class wallet_keys_unlocker + { + public: + wallet_keys_unlocker(wallet2 &w, const boost::optional<tools::password_container> &password); + wallet_keys_unlocker(wallet2 &w, bool locked, const epee::wipeable_string &password); + ~wallet_keys_unlocker(); + private: + wallet2 &w; + bool locked; + crypto::chacha_key key; + }; class i_wallet2_callback { @@ -77,6 +90,7 @@ namespace tools virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx) {} + virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason) { return boost::none; } // Light wallet callbacks virtual void on_lw_new_block(uint64_t height) {} virtual void on_lw_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} @@ -133,9 +147,11 @@ namespace tools std::deque<crypto::hash> m_blockchain; }; + class wallet_keys_unlocker; class wallet2 { friend class ::Serialization_portability_wallet_Test; + friend class wallet_keys_unlocker; public: static constexpr const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); @@ -146,28 +162,35 @@ namespace tools RefreshDefault = RefreshOptimizeCoinbase, }; + enum AskPasswordType { + AskPasswordNever = 0, + AskPasswordOnAction = 1, + AskPasswordToDecrypt = 2, + }; + static const char* tr(const char* str); static bool has_testnet_option(const boost::program_options::variables_map& vm); static bool has_stagenet_option(const boost::program_options::variables_map& vm); + static std::string device_name_option(const boost::program_options::variables_map& vm); static void init_options(boost::program_options::options_description& desc_params); //! Uses stdin and stdout. Returns a wallet2 if no errors. - static std::unique_ptr<wallet2> make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::unique_ptr<wallet2> make_from_json(const boost::program_options::variables_map& vm, bool unattended, const std::string& json_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Uses stdin and stdout. Returns a wallet2 and password for `wallet_file` if no errors. static std::pair<std::unique_ptr<wallet2>, password_container> - make_from_file(const boost::program_options::variables_map& vm, const std::string& wallet_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + make_from_file(const boost::program_options::variables_map& vm, bool unattended, const std::string& wallet_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Uses stdin and stdout. Returns a wallet2 and password for wallet with no file if no errors. - static std::pair<std::unique_ptr<wallet2>, password_container> make_new(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::pair<std::unique_ptr<wallet2>, password_container> make_new(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Just parses variables. - static std::unique_ptr<wallet2> make_dummy(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::unique_ptr<wallet2> make_dummy(const boost::program_options::variables_map& vm, bool unattended, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); - static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev); + static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds); - wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, bool restricted = false); + wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false); ~wallet2(); struct multisig_info @@ -260,12 +283,12 @@ namespace tools uint64_t m_block_height; uint64_t m_unlock_time; uint64_t m_timestamp; + bool m_coinbase; cryptonote::subaddress_index m_subaddr_index; }; struct address_tx : payment_details { - bool m_coinbase; bool m_mempool; bool m_incoming; }; @@ -477,16 +500,6 @@ namespace tools std::vector<is_out_data> additional; }; - struct key_ref - { - key_ref(tools::wallet2 &w): wallet(w) { ++refs; } - ~key_ref() { if (!--refs) wallet.clear_ringdb_key(); } - - private: - tools::wallet2 &wallet; - static std::atomic<unsigned int> refs; - }; - /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -495,7 +508,7 @@ namespace tools * \param create_address_file Whether to create an address file */ void generate(const std::string& wallet_, const epee::wipeable_string& password, - const std::string& multisig_data, bool create_address_file = false); + const epee::wipeable_string& multisig_data, bool create_address_file = false); /*! * \brief Generates a wallet or restores one. @@ -538,8 +551,9 @@ namespace tools * \param wallet_ Name of wallet file * \param password Password of wallet file * \param device_name name of HW to use + * \param create_address_file Whether to create an address file */ - void restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name); + void restore(const std::string& wallet_, const epee::wipeable_string& password, const std::string &device_name, bool create_address_file); /*! * \brief Creates a multisig wallet @@ -609,35 +623,38 @@ namespace tools /*! * \brief verifies given password is correct for default wallet keys file */ - bool verify_password(const epee::wipeable_string& password) const; + bool verify_password(const epee::wipeable_string& password); cryptonote::account_base& get_account(){return m_account;} const cryptonote::account_base& get_account()const{return m_account;} + void encrypt_keys(const crypto::chacha_key &key); + void encrypt_keys(const epee::wipeable_string &password); + void decrypt_keys(const crypto::chacha_key &key); + void decrypt_keys(const epee::wipeable_string &password); + void set_refresh_from_block_height(uint64_t height) {m_refresh_from_block_height = height;} uint64_t get_refresh_from_block_height() const {return m_refresh_from_block_height;} void explicit_refresh_from_block_height(bool expl) {m_explicit_refresh_from_block_height = expl;} bool explicit_refresh_from_block_height() const {return m_explicit_refresh_from_block_height;} - // upper_transaction_size_limit as defined below is set to - // approximately 125% of the fixed minimum allowable penalty - // free block size. TODO: fix this so that it actually takes - // into account the current median block size rather than - // the minimum block size. bool deinit(); bool init(std::string daemon_address = "http://localhost:8080", - boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_size_limit = 0, bool ssl = false); + boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, bool ssl = false, bool trusted_daemon = false); void stop() { m_run.store(false, std::memory_order_relaxed); } i_wallet2_callback* callback() const { return m_callback; } void callback(i_wallet2_callback* callback) { m_callback = callback; } + bool is_trusted_daemon() const { return m_trusted_daemon; } + void set_trusted_daemon(bool trusted) { m_trusted_daemon = trusted; } + /*! * \brief Checks if deterministic wallet */ bool is_deterministic() const; - bool get_seed(std::string& electrum_words, const epee::wipeable_string &passphrase = epee::wipeable_string()) const; + bool get_seed(epee::wipeable_string& electrum_words, const epee::wipeable_string &passphrase = epee::wipeable_string()) const; /*! * \brief Checks if light wallet. A light wallet sends view key to a server where the blockchain is scanned. @@ -659,6 +676,7 @@ namespace tools // Subaddress scheme cryptonote::account_public_address get_subaddress(const cryptonote::subaddress_index& index) const; cryptonote::account_public_address get_address() const { return get_subaddress({0,0}); } + boost::optional<cryptonote::subaddress_index> get_subaddress_index(const cryptonote::account_public_address& address) const; crypto::public_key get_subaddress_spend_public_key(const cryptonote::subaddress_index& index) const; std::vector<crypto::public_key> get_subaddress_spend_public_keys(uint32_t account, uint32_t begin, uint32_t end) const; std::string get_subaddress_as_str(const cryptonote::subaddress_index& index) const; @@ -686,13 +704,13 @@ namespace tools RefreshType get_refresh_type() const { return m_refresh_type; } cryptonote::network_type nettype() const { return m_nettype; } - bool restricted() const { return m_restricted; } bool watch_only() const { return m_watch_only; } bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const; bool has_multisig_partial_key_images() const; bool has_unknown_key_images() const; - bool get_multisig_seed(std::string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const; + bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const; bool key_on_device() const { return m_key_on_device; } + bool reconnect_device(); // locked & unlocked balance of given or current subaddress account uint64_t balance(uint32_t subaddr_index_major) const; @@ -704,18 +722,12 @@ namespace tools uint64_t balance_all() const; uint64_t unlocked_balance_all() const; template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon); - template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); - template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); void transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof); + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type); void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); @@ -737,18 +749,17 @@ namespace tools bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const; bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL); bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func); - std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); // pass subaddr_indices by value on purpose - std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); + std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); // pass subaddr_indices by value on purpose + std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); + std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); bool sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto::hash> &txids); bool sign_multisig_tx_to_file(multisig_tx_set &exported_txs, const std::string &filename, std::vector<crypto::hash> &txids); - std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon); - void discard_unmixable_outputs(bool trusted_daemon); + std::vector<pending_tx> create_unmixable_sweep_transactions(); + void discard_unmixable_outputs(); bool check_connection(uint32_t *version = NULL, uint32_t timeout = 200000); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; @@ -903,8 +914,8 @@ namespace tools void auto_refresh(bool r) { m_auto_refresh = r; } bool confirm_missing_payment_id() const { return m_confirm_missing_payment_id; } void confirm_missing_payment_id(bool always) { m_confirm_missing_payment_id = always; } - bool ask_password() const { return m_ask_password; } - void ask_password(bool always) { m_ask_password = always; } + AskPasswordType ask_password() const { return m_ask_password; } + void ask_password(AskPasswordType ask) { m_ask_password = ask; } void set_min_output_count(uint32_t count) { m_min_output_count = count; } uint32_t get_min_output_count() const { return m_min_output_count; } void set_min_output_value(uint64_t value) { m_min_output_value = value; } @@ -929,8 +940,11 @@ namespace tools void ignore_fractional_outputs(bool value) { m_ignore_fractional_outputs = value; } bool confirm_non_default_ring_size() const { return m_confirm_non_default_ring_size; } void confirm_non_default_ring_size(bool always) { m_confirm_non_default_ring_size = always; } + const std::string & device_name() const { return m_device_name; } + void device_name(const std::string & device_name) { m_device_name = device_name; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; + void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); void check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); std::string get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message); @@ -984,10 +998,10 @@ namespace tools */ uint64_t get_approximate_blockchain_height() const; uint64_t estimate_blockchain_height(); - std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon); + std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct); std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const; - std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon); - std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon); + std::vector<size_t> select_available_unmixable_outputs(); + std::vector<size_t> select_available_mixable_outputs(); size_t pop_best_value_from(const transfer_container &transfers, std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; size_t pop_best_value(std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; @@ -1053,9 +1067,12 @@ namespace tools void update_pool_state(bool refreshed = false); void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes); + std::string encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated = true) const; + std::string encrypt(const epee::span<char> &span, const crypto::secret_key &skey, bool authenticated = true) const; std::string encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated = true) const; + std::string encrypt(const epee::wipeable_string &plaintext, const crypto::secret_key &skey, bool authenticated = true) const; std::string encrypt_with_view_secret_key(const std::string &plaintext, bool authenticated = true) const; - std::string decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated = true) const; + template<typename T=std::string> T decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated = true) const; std::string decrypt_with_view_secret_key(const std::string &ciphertext, bool authenticated = true) const; std::string make_uri(const std::string &address, const std::string &payment_id, uint64_t amount, const std::string &tx_description, const std::string &recipient_name, std::string &error) const; @@ -1066,13 +1083,18 @@ namespace tools bool is_synced() const; std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(const std::vector<std::pair<double, double>> &fee_levels); - std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees); + std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees); uint64_t get_fee_multiplier(uint32_t priority, int fee_algorithm = -1) const; - uint64_t get_per_kb_fee() const; + uint64_t get_base_fee() const; + uint64_t get_fee_quantization_mask() const; + uint64_t get_min_ring_size() const; + uint64_t get_max_ring_size() const; uint64_t adjust_mixin(uint64_t mixin) const; uint32_t adjust_priority(uint32_t priority); + bool is_unattended() const { return m_unattended; } + // Light wallet specific functions // fetch unspent outs from lw node and store in m_transfers void light_wallet_get_unspent_outs(); @@ -1141,10 +1163,16 @@ namespace tools bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); bool find_and_save_rings(bool force = true); - bool blackball_output(const crypto::public_key &output); - bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false); - bool unblackball_output(const crypto::public_key &output); - bool is_output_blackballed(const crypto::public_key &output) const; + bool blackball_output(const std::pair<uint64_t, uint64_t> &output); + bool set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add = false); + bool unblackball_output(const std::pair<uint64_t, uint64_t> &output); + bool is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const; + + bool lock_keys_file(); + bool unlock_keys_file(); + bool is_keys_file_locked() const; + + void change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password); private: /*! @@ -1172,7 +1200,7 @@ namespace tools void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error); void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added); - uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const; + uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); void process_outgoing(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received, uint32_t subaddr_account, const std::set<uint32_t>& subaddr_indices); @@ -1180,14 +1208,15 @@ namespace tools void generate_genesis(cryptonote::block& b) const; void check_genesis(const crypto::hash& genesis_hash) const; //throws bool generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const; + void generate_chacha_key_from_password(const epee::wipeable_string &pass, crypto::chacha_key &key) const; crypto::hash get_payment_id(const pending_tx &ptx) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, tx_scan_info_t &tx_scan_info) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const; void check_acc_out_precomp_once(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; - uint64_t get_upper_transaction_size_limit() const; + uint64_t get_upper_transaction_weight_limit() const; std::vector<uint64_t> get_unspent_amounts_vector() const; - uint64_t get_dynamic_per_kb_fee_estimate() const; + uint64_t get_dynamic_base_fee_estimate() const; float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const; std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices) const; void set_spent(size_t idx, uint64_t height); @@ -1197,7 +1226,7 @@ namespace tools crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; std::vector<size_t> get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const; - void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) const; + void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); void trim_hashchain(); crypto::key_image get_multisig_composite_key_image(size_t n) const; rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const crypto::public_key &ignore, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; @@ -1209,15 +1238,17 @@ namespace tools bool remove_rings(const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); crypto::chacha_key get_ringdb_key(); - void cache_ringdb_key(); - void clear_ringdb_key(); + void setup_keys(const epee::wipeable_string &password); - bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); + bool get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); uint64_t get_segregation_fork_height() const; void cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const; + void setup_new_blockchain(); + void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file); + cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; std::string m_daemon_address; @@ -1243,7 +1274,7 @@ namespace tools std::unordered_map<std::string, std::string> m_attributes; std::vector<tools::wallet2::address_book_row> m_address_book; std::pair<std::map<std::string, std::string>, std::vector<std::string>> m_account_tags; - uint64_t m_upper_transaction_size_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value + uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; @@ -1251,10 +1282,11 @@ namespace tools boost::mutex m_daemon_rpc_mutex; + bool m_trusted_daemon; i_wallet2_callback* m_callback; bool m_key_on_device; cryptonote::network_type m_nettype; - bool m_restricted; + uint64_t m_kdf_rounds; std::string seed_language; /*!< Language of the mnemonics (seed). */ bool is_old_file_format; /*!< Whether the wallet file is of an old file format */ bool m_watch_only; /*!< no spend key */ @@ -1275,7 +1307,7 @@ namespace tools bool m_explicit_refresh_from_block_height; bool m_confirm_missing_payment_id; bool m_confirm_non_default_ring_size; - bool m_ask_password; + AskPasswordType m_ask_password; uint32_t m_min_output_count; uint64_t m_min_output_value; bool m_merge_destinations; @@ -1291,6 +1323,7 @@ namespace tools NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; size_t m_subaddress_lookahead_major, m_subaddress_lookahead_minor; + std::string m_device_name; // Light wallet bool m_light_wallet; /* sends view key to daemon for scanning */ @@ -1313,6 +1346,11 @@ namespace tools uint64_t m_last_block_reward; std::unique_ptr<tools::file_locker> m_keys_file_locker; + + crypto::chacha_key m_cache_key; + boost::optional<epee::wipeable_string> m_encrypt_keys_after_refresh; + + bool m_unattended; }; } BOOST_CLASS_VERSION(tools::wallet2, 25) @@ -1320,7 +1358,7 @@ BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) -BOOST_CLASS_VERSION(tools::wallet2::payment_details, 3) +BOOST_CLASS_VERSION(tools::wallet2::payment_details, 4) BOOST_CLASS_VERSION(tools::wallet2::pool_payment_details, 1) BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8) BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) @@ -1587,16 +1625,24 @@ namespace boost a & x.m_timestamp; if (ver < 2) { + x.m_coinbase = false; x.m_subaddr_index = {}; return; } a & x.m_subaddr_index; if (ver < 3) { + x.m_coinbase = false; x.m_fee = 0; return; } a & x.m_fee; + if (ver < 4) + { + x.m_coinbase = false; + return; + } + a & x.m_coinbase; } template <class Archive> @@ -1779,198 +1825,4 @@ namespace tools //---------------------------------------------------------------------------------------------------- } //---------------------------------------------------------------------------------------------------- - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon) - { - pending_tx ptx; - cryptonote::transaction tx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx, trusted_daemon); - } - - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, bool trusted_daemon) - { - using namespace cryptonote; - // throw if attempting a transaction with no destinations - THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - - THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); - uint64_t needed_money = fee; - - // calculate total amount being sent to all destinations - // throw if total amount overflows uint64_t - for(auto& dt: dsts) - { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); - needed_money += dt.amount; - THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); - } - - // randomly select inputs for transaction - // throw if requested send amount is greater than (unlocked) amount available to send - std::vector<size_t> selected_transfers; - uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers, trusted_daemon); - THROW_WALLET_EXCEPTION_IF(found_money < needed_money, error::not_enough_unlocked_money, found_money, needed_money - fee, fee); - - uint32_t subaddr_account = m_transfers[*selected_transfers.begin()].m_subaddr_index.major; - for (auto i = ++selected_transfers.begin(); i != selected_transfers.end(); ++i) - THROW_WALLET_EXCEPTION_IF(subaddr_account != *i, error::wallet_internal_error, "the tx uses funds from multiple accounts"); - - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - typedef cryptonote::tx_source_entry::output_entry tx_output_entry; - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - if(fake_outputs_count) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req = AUTO_VAL_INIT(req); - req.outs_count = fake_outputs_count + 1;// add one to make possible (if need) to skip real output key - for(size_t idx: selected_transfers) - { - const transfer_container::const_iterator it = m_transfers.begin() + idx; - THROW_WALLET_EXCEPTION_IF(it->m_tx.vout.size() <= it->m_internal_output_index, error::wallet_internal_error, - "m_internal_output_index = " + std::to_string(it->m_internal_output_index) + - " is greater or equal to outputs count = " + std::to_string(it->m_tx.vout.size())); - req.amounts.push_back(it->amount()); - } - - m_daemon_rpc_mutex.lock(); - bool r = epee::net_utils::invoke_http_bin("/getrandom_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); - THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != selected_transfers.size(), error::wallet_internal_error, - "daemon returned wrong response for getrandom_outs.bin, wrong amounts count = " + - std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(selected_transfers.size())); - - std::unordered_map<uint64_t, uint64_t> scanty_outs; - for(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& amount_outs: daemon_resp.outs) - { - if (amount_outs.outs.size() < fake_outputs_count) - { - scanty_outs[amount_outs.amount] = amount_outs.outs.size(); - } - } - THROW_WALLET_EXCEPTION_IF(!scanty_outs.empty(), error::not_enough_outs_to_mix, scanty_outs, fake_outputs_count); - } - - //prepare inputs - size_t i = 0; - std::vector<cryptonote::tx_source_entry> sources; - for(size_t idx: selected_transfers) - { - sources.resize(sources.size()+1); - cryptonote::tx_source_entry& src = sources.back(); - const transfer_details& td = m_transfers[idx]; - src.amount = td.amount(); - src.rct = false; - //paste mixin transaction - if(daemon_resp.outs.size()) - { - daemon_resp.outs[i].outs.sort([](const out_entry& a, const out_entry& b){return a.global_amount_index < b.global_amount_index;}); - for(out_entry& daemon_oe: daemon_resp.outs[i].outs) - { - if(td.m_global_output_index == daemon_oe.global_amount_index) - continue; - tx_output_entry oe; - oe.first = daemon_oe.global_amount_index; - oe.second.dest = rct::pk2rct(daemon_oe.out_key); - oe.second.mask = rct::identity(); - src.outputs.push_back(oe); - if(src.outputs.size() >= fake_outputs_count) - break; - } - } - - //paste real transaction to the random index - auto it_to_insert = std::find_if(src.outputs.begin(), src.outputs.end(), [&](const tx_output_entry& a) - { - return a.first >= td.m_global_output_index; - }); - //size_t real_index = src.outputs.size() ? (rand() % src.outputs.size() ):0; - tx_output_entry real_oe; - real_oe.first = td.m_global_output_index; - real_oe.second.dest = rct::pk2rct(boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key); - real_oe.second.mask = rct::identity(); - auto interted_it = src.outputs.insert(it_to_insert, real_oe); - src.real_out_tx_key = get_tx_pub_key_from_extra(td.m_tx); - src.real_output = interted_it - src.outputs.begin(); - src.real_output_in_tx_index = td.m_internal_output_index; - src.multisig_kLRki = rct::multisig_kLRki({rct::zero(), rct::zero(), rct::zero(), rct::zero()}); - detail::print_source_entry(src); - ++i; - } - - cryptonote::tx_destination_entry change_dts = AUTO_VAL_INIT(change_dts); - if (needed_money < found_money) - { - change_dts.addr = get_subaddress({subaddr_account, 0}); - change_dts.amount = found_money - needed_money; - } - - std::vector<cryptonote::tx_destination_entry> splitted_dsts, dust_dsts; - uint64_t dust = 0; - destination_split_strategy(dsts, change_dts, dust_policy.dust_threshold, splitted_dsts, dust_dsts); - for(auto& d: dust_dsts) { - THROW_WALLET_EXCEPTION_IF(dust_policy.dust_threshold < d.amount, error::wallet_internal_error, "invalid dust value: dust = " + - std::to_string(d.amount) + ", dust_threshold = " + std::to_string(dust_policy.dust_threshold)); - } - for(auto& d: dust_dsts) { - if (!dust_policy.add_to_fee) - splitted_dsts.push_back(cryptonote::tx_destination_entry(d.amount, dust_policy.addr_for_dust, d.is_subaddress)); - dust += d.amount; - } - - crypto::secret_key tx_key; - std::vector<crypto::secret_key> additional_tx_keys; - rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); - - std::string key_images; - bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool - { - CHECKED_GET_SPECIFIC_VARIANT(s_e, const txin_to_key, in, false); - key_images += boost::to_string(in.k_image) + " "; - return true; - }); - THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx); - - bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key - || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key); - - if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust; - - ptx.key_images = key_images; - ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee); - ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0); - ptx.dust_added_to_fee = dust_policy.add_to_fee; - ptx.tx = tx; - ptx.change_dts = change_dts; - ptx.selected_transfers = selected_transfers; - ptx.tx_key = tx_key; - ptx.additional_tx_keys = additional_tx_keys; - ptx.dests = dsts; - ptx.construction_data.sources = sources; - ptx.construction_data.change_dts = change_dts; - ptx.construction_data.splitted_dsts = splitted_dsts; - ptx.construction_data.selected_transfers = selected_transfers; - ptx.construction_data.extra = tx.extra; - ptx.construction_data.unlock_time = unlock_time; - ptx.construction_data.use_rct = false; - ptx.construction_data.use_bulletproofs = false; - ptx.construction_data.dests = dsts; - // record which subaddress indices are being used as inputs - ptx.construction_data.subaddr_account = subaddr_account; - ptx.construction_data.subaddr_indices.clear(); - for (size_t idx: selected_transfers) - ptx.construction_data.subaddr_indices.insert(m_transfers[idx].m_subaddr_index.minor); - } - - } diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index a629eb506..95a4e0ad6 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -109,6 +109,9 @@ namespace wallet_args std::string lang = i18n_get_language(); tools::on_startup(); +#ifdef NDEBUG + tools::disable_core_dumps(); +#endif tools::set_strict_default_file_permissions(true); epee::string_tools::set_module_name_and_folder(argv[0]); diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index e80652750..bc518d04a 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -53,6 +53,7 @@ namespace tools // wallet_not_initialized // multisig_export_needed // multisig_import_needed + // password_needed // std::logic_error // wallet_logic_error * // file_exists @@ -72,7 +73,7 @@ namespace tools // get_tx_pool_error // out_of_hashchain_bounds_error // transfer_error * - // get_random_outs_general_error + // get_outs_general_error // not_enough_unlocked_money // not_enough_money // tx_not_possible @@ -127,7 +128,7 @@ namespace tools get_blocks_error_message_index, get_hashes_error_message_index, get_out_indices_error_message_index, - get_random_outs_error_message_index + get_outs_error_message_index }; template<typename Base, int msg_index> @@ -209,6 +210,14 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct password_needed : public wallet_runtime_error + { + explicit password_needed(std::string&& loc, const std::string &msg = "Password needed") + : wallet_runtime_error(std::move(loc), msg) + { + } + }; + //---------------------------------------------------------------------------------------------------- const char* const file_error_messages[] = { "file already exists", "file not found", @@ -418,7 +427,7 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- - typedef failed_rpc_request<transfer_error, get_random_outs_error_message_index> get_random_outs_error; + typedef failed_rpc_request<transfer_error, get_outs_error_message_index> get_outs_error; //---------------------------------------------------------------------------------------------------- struct not_enough_unlocked_money : public transfer_error { @@ -670,30 +679,30 @@ namespace tools //---------------------------------------------------------------------------------------------------- struct tx_too_big : public transfer_error { - explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_size_limit) + explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_weight_limit) : transfer_error(std::move(loc), "transaction is too big") , m_tx(tx) - , m_tx_size_limit(tx_size_limit) + , m_tx_weight_limit(tx_weight_limit) { } const cryptonote::transaction& tx() const { return m_tx; } - uint64_t tx_size_limit() const { return m_tx_size_limit; } + uint64_t tx_weight_limit() const { return m_tx_weight_limit; } std::string to_string() const { std::ostringstream ss; cryptonote::transaction tx = m_tx; ss << transfer_error::to_string() << - ", tx_size_limit = " << m_tx_size_limit << - ", tx size = " << get_object_blobsize(m_tx) << + ", tx_weight_limit = " << m_tx_weight_limit << + ", tx weight = " << get_transaction_weight(m_tx) << ", tx:\n" << cryptonote::obj_to_json_str(tx); return ss.str(); } private: cryptonote::transaction m_tx; - uint64_t m_tx_size_limit; + uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- struct zero_destination : public transfer_error diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index b9cf99635..86b46b173 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -51,6 +51,7 @@ using namespace epee; #include "mnemonics/electrum-words.h" #include "rpc/rpc_args.h" #include "rpc/core_rpc_server_commands_defs.h" +#include "daemonizer/daemonizer.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.rpc" @@ -59,7 +60,7 @@ namespace { const command_line::arg_descriptor<std::string, true> arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"}; const command_line::arg_descriptor<bool> arg_disable_rpc_login = {"disable-rpc-login", "Disable HTTP authentication for RPC connections served by this process"}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", "Enable commands which rely on a trusted daemon", false}; + const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false}; const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"}; const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false}; @@ -99,7 +100,7 @@ namespace tools } //------------------------------------------------------------------------------------------------------------------------------ - wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_trusted_daemon(false), m_vm(NULL) + wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_restricted(false), m_vm(NULL) { } //------------------------------------------------------------------------------------------------------------------------------ @@ -119,7 +120,7 @@ namespace tools m_stop = false; m_net_server.add_idle_handler([this](){ try { - if (m_wallet) m_wallet->refresh(m_trusted_daemon); + if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon()); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } @@ -155,30 +156,18 @@ namespace tools return false; m_vm = vm; - tools::wallet2 *walvars; - std::unique_ptr<tools::wallet2> tmpwal; - if (m_wallet) - walvars = m_wallet; - else - { - tmpwal = tools::wallet2::make_dummy(*m_vm, password_prompter); - walvars = tmpwal.get(); - } boost::optional<epee::net_utils::http::login> http_login{}; std::string bind_port = command_line::get_arg(*m_vm, arg_rpc_bind_port); const bool disable_auth = command_line::get_arg(*m_vm, arg_disable_rpc_login); - m_trusted_daemon = command_line::get_arg(*m_vm, arg_trusted_daemon); - if (!command_line::has_arg(*m_vm, arg_trusted_daemon)) + m_restricted = command_line::get_arg(*m_vm, arg_restricted); + if (!command_line::is_arg_defaulted(*m_vm, arg_wallet_dir)) { - if (tools::is_local_address(walvars->get_daemon_address())) + if (!command_line::is_arg_defaulted(*m_vm, wallet_args::arg_wallet_file())) { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; + MERROR(arg_wallet_dir.name << " and " << wallet_args::arg_wallet_file().name << " are incompatible, use only one of them"); + return false; } - } - if (command_line::has_arg(*m_vm, arg_wallet_dir)) - { m_wallet_dir = command_line::get_arg(*m_vm, arg_wallet_dir); #ifdef _WIN32 #define MKDIR(path, mode) mkdir(path) @@ -270,7 +259,7 @@ namespace tools entry.unlock_time = pd.m_unlock_time; entry.fee = pd.m_fee; entry.note = m_wallet->get_tx_note(pd.m_tx_hash); - entry.type = "in"; + entry.type = pd.m_coinbase ? "block" : "in"; entry.subaddr_index = pd.m_subaddr_index; entry.address = m_wallet->get_subaddress_as_str(pd.m_subaddr_index); set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); @@ -355,14 +344,20 @@ namespace tools std::map<uint32_t, uint64_t> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(req.account_index); std::vector<tools::wallet2::transfer_details> transfers; m_wallet->get_transfers(transfers); - for (const auto& i : balance_per_subaddress) + std::set<uint32_t> address_indices = req.address_indices; + if (address_indices.empty()) + { + for (const auto& i : balance_per_subaddress) + address_indices.insert(i.first); + } + for (uint32_t i : address_indices) { wallet_rpc::COMMAND_RPC_GET_BALANCE::per_subaddress_info info; - info.address_index = i.first; + info.address_index = i; cryptonote::subaddress_index index = {req.account_index, info.address_index}; info.address = m_wallet->get_subaddress_as_str(index); - info.balance = i.second; - info.unlocked_balance = unlocked_balance_per_subaddress[i.first]; + info.balance = balance_per_subaddress[i]; + info.unlocked_balance = unlocked_balance_per_subaddress[i]; info.label = m_wallet->get_subaddress_label(index); info.num_unspent_outputs = std::count_if(transfers.begin(), transfers.end(), [&](const tools::wallet2::transfer_details& td) { return !td.m_spent && td.m_subaddr_index == index; }); res.per_subaddress.push_back(info); @@ -416,6 +411,27 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + cryptonote::address_parse_info info; + if(!get_account_address_from_str(info, m_wallet->nettype(), req.address)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Invalid address"; + return false; + } + auto index = m_wallet->get_subaddress_index(info.address); + if (!index) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Address doesn't belong to the wallet"; + return false; + } + res.index = *index; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); @@ -744,10 +760,10 @@ namespace tools { if (get_tx_key) { - std::string s = epee::string_tools::pod_to_hex(ptx.tx_key); + epee::wipeable_string s = epee::to_hex::wipeable_string(ptx.tx_key); for (const crypto::secret_key& additional_tx_key : ptx.additional_tx_keys) - s += epee::string_tools::pod_to_hex(additional_tx_key); - fill(tx_key, s); + s += epee::to_hex::wipeable_string(additional_tx_key); + fill(tx_key, std::string(s.data(), s.size())); } // Compute amount leaving wallet in tx. By convention dests does not include change outputs fill(amount, total_amount(ptx)); @@ -803,7 +819,7 @@ namespace tools LOG_PRINT_L3("on_transfer starts"); if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -828,7 +844,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); if (ptx_vector.empty()) { @@ -863,7 +879,7 @@ namespace tools std::vector<uint8_t> extra; if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -889,7 +905,7 @@ namespace tools } uint32_t priority = m_wallet->adjust_priority(req.priority); LOG_PRINT_L2("on_transfer_split calling create_transactions_2"); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); LOG_PRINT_L2("on_transfer_split called create_transactions_2"); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, @@ -906,7 +922,7 @@ namespace tools bool wallet_rpc_server::on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -981,7 +997,7 @@ namespace tools bool wallet_rpc_server::on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1041,7 +1057,7 @@ namespace tools bool wallet_rpc_server::on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1050,7 +1066,7 @@ namespace tools try { - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); @@ -1069,7 +1085,7 @@ namespace tools std::vector<uint8_t> extra; if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1098,7 +1114,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); @@ -1117,7 +1133,7 @@ namespace tools std::vector<uint8_t> extra; if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1154,7 +1170,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra); if (ptx_vector.empty()) { @@ -1255,7 +1271,39 @@ namespace tools } } - res.integrated_address = m_wallet->get_integrated_address_as_str(payment_id); + if (req.standard_address.empty()) + { + res.integrated_address = m_wallet->get_integrated_address_as_str(payment_id); + } + else + { + cryptonote::address_parse_info info; + if(!get_account_address_from_str(info, m_wallet->nettype(), req.standard_address)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Invalid address"; + return false; + } + if (info.is_subaddress) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Subaddress shouldn't be used"; + return false; + } + if (info.has_payment_id) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + er.message = "Already integrated address"; + return false; + } + if (req.payment_id.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment ID shouldn't be left unspecified"; + return false; + } + res.integrated_address = get_account_integrated_address_as_str(m_wallet->nettype(), info.address, payment_id); + } res.payment_id = epee::string_tools::pod_to_hex(payment_id); return true; } @@ -1301,7 +1349,7 @@ namespace tools bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1491,7 +1539,6 @@ namespace tools rpc_transfers.spent = td.m_spent; rpc_transfers.global_index = td.m_global_output_index; rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(td.m_txid); - rpc_transfers.tx_size = txBlob.size(); rpc_transfers.subaddr_index = td.m_subaddr_index.minor; rpc_transfers.key_image = req.verbose && td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; res.transfers.push_back(rpc_transfers); @@ -1504,7 +1551,7 @@ namespace tools bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1513,19 +1560,23 @@ namespace tools if (req.key_type.compare("mnemonic") == 0) { - if (!m_wallet->get_seed(res.key)) + epee::wipeable_string seed; + if (!m_wallet->get_seed(seed)) { er.message = "The wallet is non-deterministic. Cannot display seed."; return false; } + res.key = std::string(seed.data(), seed.size()); // send to the network, then wipe RAM :D } else if(req.key_type.compare("view_key") == 0) { - res.key = string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key); + epee::wipeable_string key = epee::to_hex::wipeable_string(m_wallet->get_account().get_keys().m_view_secret_key); + res.key = std::string(key.data(), key.size()); } else if(req.key_type.compare("spend_key") == 0) { - res.key = string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_spend_secret_key); + epee::wipeable_string key = epee::to_hex::wipeable_string(m_wallet->get_account().get_keys().m_spend_secret_key); + res.key = std::string(key.data(), key.size()); } else { @@ -1539,7 +1590,7 @@ namespace tools bool wallet_rpc_server::on_rescan_blockchain(const wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::request& req, wallet_rpc::COMMAND_RPC_RESCAN_BLOCKCHAIN::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1561,7 +1612,7 @@ namespace tools bool wallet_rpc_server::on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1575,7 +1626,7 @@ namespace tools bool wallet_rpc_server::on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1610,7 +1661,7 @@ namespace tools bool wallet_rpc_server::on_stop_wallet(const wallet_rpc::COMMAND_RPC_STOP_WALLET::request& req, wallet_rpc::COMMAND_RPC_STOP_WALLET::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1633,7 +1684,7 @@ namespace tools bool wallet_rpc_server::on_set_tx_notes(const wallet_rpc::COMMAND_RPC_SET_TX_NOTES::request& req, wallet_rpc::COMMAND_RPC_SET_TX_NOTES::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1705,7 +1756,7 @@ namespace tools bool wallet_rpc_server::on_set_attribute(const wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_SET_ATTRIBUTE::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1720,7 +1771,7 @@ namespace tools bool wallet_rpc_server::on_get_attribute(const wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::request& req, wallet_rpc::COMMAND_RPC_GET_ATTRIBUTE::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -1751,11 +1802,11 @@ namespace tools return false; } - std::ostringstream oss; - oss << epee::string_tools::pod_to_hex(tx_key); + epee::wipeable_string s; + s += epee::to_hex::wipeable_string(tx_key); for (size_t i = 0; i < additional_tx_keys.size(); ++i) - oss << epee::string_tools::pod_to_hex(additional_tx_keys[i]); - res.tx_key = oss.str(); + s += epee::to_hex::wipeable_string(additional_tx_keys[i]); + res.tx_key = std::string(s.data(), s.size()); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1771,26 +1822,33 @@ namespace tools return false; } - std::string tx_key_str = req.tx_key; + epee::wipeable_string tx_key_str = req.tx_key; + if (tx_key_str.size() < 64 || tx_key_str.size() % 64) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY; + er.message = "Tx key has invalid format"; + return false; + } + const char *data = tx_key_str.data(); crypto::secret_key tx_key; - if (!epee::string_tools::hex_to_pod(tx_key_str.substr(0, 64), tx_key)) + if (!epee::wipeable_string(data, 64).hex_to_pod(unwrap(unwrap(tx_key)))) { er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY; er.message = "Tx key has invalid format"; return false; } - tx_key_str = tx_key_str.substr(64); + size_t offset = 64; std::vector<crypto::secret_key> additional_tx_keys; - while (!tx_key_str.empty()) + while (offset < tx_key_str.size()) { additional_tx_keys.resize(additional_tx_keys.size() + 1); - if (!epee::string_tools::hex_to_pod(tx_key_str.substr(0, 64), additional_tx_keys.back())) + if (!epee::wipeable_string(data + offset, 64).hex_to_pod(unwrap(unwrap(additional_tx_keys.back())))) { er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY; er.message = "Tx key has invalid format"; return false; } - tx_key_str = tx_key_str.substr(64); + offset += 64; } cryptonote::address_parse_info info; @@ -1996,7 +2054,7 @@ namespace tools bool wallet_rpc_server::on_get_transfers(const wallet_rpc::COMMAND_RPC_GET_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFERS::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2062,7 +2120,7 @@ namespace tools bool wallet_rpc_server::on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2146,7 +2204,7 @@ namespace tools bool wallet_rpc_server::on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2175,7 +2233,7 @@ namespace tools bool wallet_rpc_server::on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2235,13 +2293,13 @@ namespace tools bool wallet_rpc_server::on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; return false; } - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2346,7 +2404,7 @@ namespace tools bool wallet_rpc_server::on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2421,7 +2479,7 @@ namespace tools bool wallet_rpc_server::on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2444,10 +2502,32 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + try + { + m_wallet->refresh(m_wallet->is_trusted_daemon(), req.start_height, res.blocks_fetched, res.received_money); + return true; + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2469,7 +2549,7 @@ namespace tools bool wallet_rpc_server::on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2575,7 +2655,7 @@ namespace tools command_line::add_arg(desc, arg_password); po::store(po::parse_command_line(argc, argv, desc), vm2); } - std::unique_ptr<tools::wallet2> wal = tools::wallet2::make_new(vm2, nullptr).first; + std::unique_ptr<tools::wallet2> wal = tools::wallet2::make_new(vm2, true, nullptr).first; if (!wal) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -2603,8 +2683,20 @@ namespace tools er.message = "Failed to generate wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } @@ -2649,7 +2741,7 @@ namespace tools } std::unique_ptr<tools::wallet2> wal = nullptr; try { - wal = tools::wallet2::make_from_file(vm2, wallet_file, nullptr).first; + wal = tools::wallet2::make_from_file(vm2, true, wallet_file, nullptr).first; } catch (const std::exception& e) { @@ -2661,12 +2753,74 @@ namespace tools er.message = "Failed to open wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + m_wallet = NULL; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->verify_password(req.old_password)) + { + try + { + m_wallet->rewrite(m_wallet->get_wallet_file(), req.new_password); + m_wallet->store(); + LOG_PRINT_L0("Wallet password changed."); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + } + else + { + er.code = WALLET_RPC_ERROR_CODE_INVALID_PASSWORD; + er.message = "Invalid original password."; + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code) { try { @@ -2754,7 +2908,7 @@ namespace tools bool wallet_rpc_server::on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2780,7 +2934,7 @@ namespace tools bool wallet_rpc_server::on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2817,7 +2971,7 @@ namespace tools bool wallet_rpc_server::on_export_multisig(const wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_EXPORT_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2857,7 +3011,7 @@ namespace tools bool wallet_rpc_server::on_import_multisig(const wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IMPORT_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2908,7 +3062,7 @@ namespace tools return false; } - if (m_trusted_daemon) + if (m_wallet->is_trusted_daemon()) { try { @@ -2930,7 +3084,7 @@ namespace tools bool wallet_rpc_server::on_finalize_multisig(const wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_FINALIZE_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -2981,7 +3135,7 @@ namespace tools bool wallet_rpc_server::on_sign_multisig(const wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SIGN_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -3050,7 +3204,7 @@ namespace tools bool wallet_rpc_server::on_submit_multisig(const wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (m_wallet->restricted()) + if (m_restricted) { er.code = WALLET_RPC_ERROR_CODE_DENIED; er.message = "Command unavailable in restricted mode."; @@ -3121,23 +3275,185 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ } +class t_daemon +{ +private: + const boost::program_options::variables_map& vm; + +public: + t_daemon(boost::program_options::variables_map const & _vm) + : vm(_vm) + { + } + + bool run() + { + std::unique_ptr<tools::wallet2> wal; + try + { + const bool testnet = tools::wallet2::has_testnet_option(vm); + const bool stagenet = tools::wallet2::has_stagenet_option(vm); + if (testnet && stagenet) + { + MERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --testnet and --stagenet")); + return false; + } + + const auto arg_wallet_file = wallet_args::arg_wallet_file(); + const auto arg_from_json = wallet_args::arg_generate_from_json(); + + const auto wallet_file = command_line::get_arg(vm, arg_wallet_file); + const auto from_json = command_line::get_arg(vm, arg_from_json); + const auto wallet_dir = command_line::get_arg(vm, arg_wallet_dir); + const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password); + const auto password_prompt = prompt_for_password ? password_prompter : nullptr; + + if(!wallet_file.empty() && !from_json.empty()) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --wallet-file and --generate-from-json")); + return false; + } + + if (!wallet_dir.empty()) + { + wal = NULL; + goto just_dir; + } + + if (wallet_file.empty() && from_json.empty()) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Must specify --wallet-file or --generate-from-json or --wallet-dir")); + return false; + } + + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Loading wallet...")); + if(!wallet_file.empty()) + { + wal = tools::wallet2::make_from_file(vm, true, wallet_file, password_prompt).first; + } + else + { + try + { + wal = tools::wallet2::make_from_json(vm, true, from_json, password_prompt); + } + catch (const std::exception &e) + { + MERROR("Error creating wallet: " << e.what()); + return false; + } + } + if (!wal) + { + return false; + } + + bool quit = false; + tools::signal_handler::install([&wal, &quit](int) { + assert(wal); + quit = true; + wal->stop(); + }); + + wal->refresh(wal->is_trusted_daemon()); + // if we ^C during potentially length load/refresh, there's no server loop yet + if (quit) + { + MINFO(tools::wallet_rpc_server::tr("Saving wallet...")); + wal->store(); + MINFO(tools::wallet_rpc_server::tr("Successfully saved")); + return false; + } + MINFO(tools::wallet_rpc_server::tr("Successfully loaded")); + } + catch (const std::exception& e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Wallet initialization failed: ") << e.what()); + return false; + } + just_dir: + tools::wallet_rpc_server wrpc; + if (wal) wrpc.set_wallet(wal.release()); + bool r = wrpc.init(&vm); + CHECK_AND_ASSERT_MES(r, false, tools::wallet_rpc_server::tr("Failed to initialize wallet RPC server")); + tools::signal_handler::install([&wrpc](int) { + wrpc.send_stop_signal(); + }); + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Starting wallet RPC server")); + try + { + wrpc.run(); + } + catch (const std::exception &e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Failed to run wallet: ") << e.what()); + return false; + } + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Stopped wallet RPC server")); + try + { + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Saving wallet...")); + wrpc.stop(); + LOG_PRINT_L0(tools::wallet_rpc_server::tr("Successfully saved")); + } + catch (const std::exception& e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Failed to save wallet: ") << e.what()); + return false; + } + return true; + } +}; + +class t_executor final +{ +public: + static std::string const NAME; + + std::string const & name() + { + return NAME; + } + + t_daemon create_daemon(boost::program_options::variables_map const & vm) + { + return t_daemon(vm); + } + + bool run_non_interactive(boost::program_options::variables_map const & vm) + { + return t_daemon(vm).run(); + } + + bool run_interactive(boost::program_options::variables_map const & vm) + { + return t_daemon(vm).run(); + } +}; + +std::string const t_executor::NAME = "Wallet RPC Daemon"; + int main(int argc, char** argv) { namespace po = boost::program_options; const auto arg_wallet_file = wallet_args::arg_wallet_file(); const auto arg_from_json = wallet_args::arg_generate_from_json(); + po::options_description hidden_options("Hidden"); + po::options_description desc_params(wallet_args::tr("Wallet options")); tools::wallet2::init_options(desc_params); command_line::add_arg(desc_params, arg_rpc_bind_port); command_line::add_arg(desc_params, arg_disable_rpc_login); - command_line::add_arg(desc_params, arg_trusted_daemon); + command_line::add_arg(desc_params, arg_restricted); cryptonote::rpc_args::init_options(desc_params); command_line::add_arg(desc_params, arg_wallet_file); command_line::add_arg(desc_params, arg_from_json); command_line::add_arg(desc_params, arg_wallet_dir); command_line::add_arg(desc_params, arg_prompt_for_password); + daemonizer::init_options(hidden_options, desc_params); + boost::optional<po::variables_map> vm; bool should_terminate = false; std::tie(vm, should_terminate) = wallet_args::main( @@ -3159,115 +3475,5 @@ int main(int argc, char** argv) { return 0; } - std::unique_ptr<tools::wallet2> wal; - try - { - const bool testnet = tools::wallet2::has_testnet_option(*vm); - const bool stagenet = tools::wallet2::has_stagenet_option(*vm); - if (testnet && stagenet) - { - MERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --testnet and --stagenet")); - return 1; - } - - const auto wallet_file = command_line::get_arg(*vm, arg_wallet_file); - const auto from_json = command_line::get_arg(*vm, arg_from_json); - const auto wallet_dir = command_line::get_arg(*vm, arg_wallet_dir); - const auto prompt_for_password = command_line::get_arg(*vm, arg_prompt_for_password); - const auto password_prompt = prompt_for_password ? password_prompter : nullptr; - - if(!wallet_file.empty() && !from_json.empty()) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Can't specify more than one of --wallet-file and --generate-from-json")); - return 1; - } - - if (!wallet_dir.empty()) - { - wal = NULL; - goto just_dir; - } - - if (wallet_file.empty() && from_json.empty()) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Must specify --wallet-file or --generate-from-json or --wallet-dir")); - return 1; - } - - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Loading wallet...")); - if(!wallet_file.empty()) - { - wal = tools::wallet2::make_from_file(*vm, wallet_file, password_prompt).first; - } - else - { - try - { - wal = tools::wallet2::make_from_json(*vm, from_json, password_prompt); - } - catch (const std::exception &e) - { - MERROR("Error creating wallet: " << e.what()); - return 1; - } - } - if (!wal) - { - return 1; - } - - bool quit = false; - tools::signal_handler::install([&wal, &quit](int) { - assert(wal); - quit = true; - wal->stop(); - }); - - wal->refresh(command_line::get_arg(*vm, arg_trusted_daemon)); - // if we ^C during potentially length load/refresh, there's no server loop yet - if (quit) - { - MINFO(tools::wallet_rpc_server::tr("Saving wallet...")); - wal->store(); - MINFO(tools::wallet_rpc_server::tr("Successfully saved")); - return 1; - } - MINFO(tools::wallet_rpc_server::tr("Successfully loaded")); - } - catch (const std::exception& e) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Wallet initialization failed: ") << e.what()); - return 1; - } -just_dir: - tools::wallet_rpc_server wrpc; - if (wal) wrpc.set_wallet(wal.release()); - bool r = wrpc.init(&(vm.get())); - CHECK_AND_ASSERT_MES(r, 1, tools::wallet_rpc_server::tr("Failed to initialize wallet RPC server")); - tools::signal_handler::install([&wrpc](int) { - wrpc.send_stop_signal(); - }); - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Starting wallet RPC server")); - try - { - wrpc.run(); - } - catch (const std::exception &e) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Failed to run wallet: ") << e.what()); - return 1; - } - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Stopped wallet RPC server")); - try - { - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Saving wallet...")); - wrpc.stop(); - LOG_PRINT_L0(tools::wallet_rpc_server::tr("Successfully saved")); - } - catch (const std::exception& e) - { - LOG_ERROR(tools::wallet_rpc_server::tr("Failed to save wallet: ") << e.what()); - return 1; - } - return 0; + return daemonizer::daemonize(argc, const_cast<const char**>(argv), t_executor{}, *vm) ? 0 : 1; } diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 9cb67c593..ab7917a78 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -69,6 +69,7 @@ namespace tools BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC_WE("get_balance", on_getbalance, wallet_rpc::COMMAND_RPC_GET_BALANCE) MAP_JON_RPC_WE("get_address", on_getaddress, wallet_rpc::COMMAND_RPC_GET_ADDRESS) + MAP_JON_RPC_WE("get_address_index", on_getaddress_index, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX) MAP_JON_RPC_WE("getbalance", on_getbalance, wallet_rpc::COMMAND_RPC_GET_BALANCE) MAP_JON_RPC_WE("getaddress", on_getaddress, wallet_rpc::COMMAND_RPC_GET_ADDRESS) MAP_JON_RPC_WE("create_address", on_create_address, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS) @@ -125,12 +126,15 @@ namespace tools MAP_JON_RPC_WE("get_address_book", on_get_address_book, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("add_address_book", on_add_address_book, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) + MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) MAP_JON_RPC_WE("get_languages", on_get_languages, wallet_rpc::COMMAND_RPC_GET_LANGUAGES) MAP_JON_RPC_WE("create_wallet", on_create_wallet, wallet_rpc::COMMAND_RPC_CREATE_WALLET) MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) + MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) + MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) @@ -146,6 +150,7 @@ namespace tools //json_rpc bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er); bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er); + bool on_getaddress_index(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_INDEX::response& res, epee::json_rpc::error& er); bool on_create_address(const wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_CREATE_ADDRESS::response& res, epee::json_rpc::error& er); bool on_label_address(const wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_LABEL_ADDRESS::response& res, epee::json_rpc::error& er); bool on_get_accounts(const wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::response& res, epee::json_rpc::error& er); @@ -198,12 +203,15 @@ namespace tools bool on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); + bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er); bool on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er); bool on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er); bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); + bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); + bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); @@ -234,7 +242,7 @@ namespace tools std::string m_wallet_dir; tools::private_file rpc_login_file; std::atomic<bool> m_stop; - bool m_trusted_daemon; + bool m_restricted; const boost::program_options::variables_map *m_vm; }; } diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 1bd572add..4501cf575 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 1 +#define WALLET_RPC_VERSION_MINOR 3 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -62,8 +62,10 @@ namespace wallet_rpc struct request { uint32_t account_index; + std::set<uint32_t> address_indices; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) + KV_SERIALIZE(address_indices) END_KV_SERIALIZE_MAP() }; @@ -141,6 +143,25 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_GET_ADDRESS_INDEX + { + struct request + { + std::string address; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + cryptonote::subaddress_index index; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(index) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_CREATE_ADDRESS { struct request @@ -424,7 +445,6 @@ namespace wallet_rpc { std::string tx_hash; std::string tx_key; - std::list<std::string> amount_keys; uint64_t amount; uint64_t fee; std::string tx_blob; @@ -435,7 +455,6 @@ namespace wallet_rpc BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(tx_key) - KV_SERIALIZE(amount_keys) KV_SERIALIZE(amount) KV_SERIALIZE(fee) KV_SERIALIZE(tx_blob) @@ -747,15 +766,9 @@ namespace wallet_rpc struct response { std::string tx_hash; - std::string tx_key; - uint64_t fee; - std::string tx_blob; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_key) - KV_SERIALIZE(fee) - KV_SERIALIZE(tx_blob) END_KV_SERIALIZE_MAP() }; }; @@ -846,7 +859,6 @@ namespace wallet_rpc bool spent; uint64_t global_index; std::string tx_hash; - uint64_t tx_size; uint32_t subaddr_index; std::string key_image; @@ -855,7 +867,6 @@ namespace wallet_rpc KV_SERIALIZE(spent) KV_SERIALIZE(global_index) KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_size) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) END_KV_SERIALIZE_MAP() @@ -914,9 +925,11 @@ namespace wallet_rpc { struct request { + std::string standard_address; std::string payment_id; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(standard_address) KV_SERIALIZE(payment_id) END_KV_SERIALIZE_MAP() }; @@ -1691,6 +1704,29 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_REFRESH + { + struct request + { + uint64_t start_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(start_height, (uint64_t) 0) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t blocks_fetched; + bool received_money; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(blocks_fetched); + KV_SERIALIZE(received_money); + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_START_MINING { struct request @@ -1785,6 +1821,40 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_CLOSE_WALLET + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_CHANGE_WALLET_PASSWORD + { + struct request + { + std::string old_password; + std::string new_password; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(old_password) + KV_SERIALIZE(new_password) + END_KV_SERIALIZE_MAP() + }; + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_IS_MULTISIG { struct request |